1: %% -*- coding: utf-8 -*-
    2: %%
    3: %% %CopyrightBegin%
    4: %%
    5: %% Copyright Ericsson AB 1998-2013. All Rights Reserved.
    6: %%
    7: %% The contents of this file are subject to the Erlang Public License,
    8: %% Version 1.1, (the "License"); you may not use this file except in
    9: %% compliance with the License. You should have received a copy of the
   10: %% Erlang Public License along with this software. If not, it can be
   11: %% retrieved online at http://www.erlang.org/.
   12: %%
   13: %% Software distributed under the License is distributed on an "AS IS"
   14: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
   15: %% the License for the specific language governing rights and limitations
   16: %% under the License.
   17: %%
   18: %% %CopyrightEnd%
   19: 
   20: -module(erl_scan_SUITE).
   21: -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
   22: 	 init_per_group/2,end_per_group/2]).
   23: 
   24: -export([ error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1,
   25:           otp_10990/1, otp_10992/1]).
   26: 
   27: -import(lists, [nth/2,flatten/1]).
   28: -import(io_lib, [print/1]).
   29: 
   30: %%
   31: %% Define to run outside of test server
   32: %%
   33: %-define(STANDALONE,1).
   34: 
   35: -ifdef(STANDALONE).
   36: -compile(export_all).
   37: -define(line, put(line, ?LINE), ).
   38: -define(config(A,B),config(A,B)).
   39: -define(t, test_server).
   40: %% config(priv_dir, _) ->
   41: %%     ".";
   42: %% config(data_dir, _) ->
   43: %%     ".".
   44: -else.
   45: -include_lib("test_server/include/test_server.hrl").
   46: -export([init_per_testcase/2, end_per_testcase/2]).
   47: 
   48: init_per_testcase(_Case, Config) when is_list(Config) ->
   49:     ?line Dog=test_server:timetrap(test_server:seconds(1200)),
   50:     [{watchdog, Dog}|Config].
   51: 
   52: end_per_testcase(_Case, Config) ->
   53:     Dog=?config(watchdog, Config),
   54:     test_server:timetrap_cancel(Dog),
   55:     ok.
   56: -endif.
   57: 
   58: % Default timetrap timeout (set in init_per_testcase).
   59: -define(default_timeout, ?t:minutes(1)).
   60: 
   61: suite() -> [{ct_hooks,[ts_install_cth]}].
   62: 
   63: all() -> 
   64:     [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992].
   65: 
   66: groups() -> 
   67:     [{error, [], [error_1, error_2]}].
   68: 
   69: init_per_suite(Config) ->
   70:     Config.
   71: 
   72: end_per_suite(_Config) ->
   73:     ok.
   74: 
   75: init_per_group(_GroupName, Config) ->
   76:     Config.
   77: 
   78: end_per_group(_GroupName, Config) ->
   79:     Config.
   80: 
   81: 
   82: 
   83: error_1(doc) ->
   84:     ["(OTP-2347)"];
   85: error_1(suite) ->
   86:     [];
   87: error_1(Config) when is_list(Config) ->
   88:     ?line {error, _, _} = erl_scan:string("'a"),
   89:     ok.
   90: 
   91: error_2(doc) ->
   92:     ["Checks that format_error works on the error cases."];
   93: error_2(suite) ->
   94:     [];
   95: error_2(Config) when is_list(Config) ->
   96:     ?line lists:foreach(fun check/1, error_cases()),
   97:     ok.
   98: 
   99: error_cases() ->
  100:     ["'a",
  101:      "\"a",
  102:      "'\\",
  103:      "\"\\",
  104:      "$",
  105:      "$\\",
  106:      "2.3e",
  107:      "2.3e-",
  108:      "91#9"
  109: ].
  110: 
  111: assert_type(N, integer) when is_integer(N) ->
  112:     ok;
  113: assert_type(N, atom) when is_atom(N) ->
  114:     ok.
  115: 
  116: check(String) ->
  117:     Error = erl_scan:string(String),
  118:     check_error(Error, erl_scan).
  119: 
  120: %%% (This should be useful for all format_error functions.)
  121: check_error({error, Info, EndLine}, Module0) ->
  122:     {ErrorLine, Module, Desc} = Info,
  123:     true = (Module == Module0),
  124:     assert_type(EndLine, integer),
  125:     assert_type(ErrorLine, integer),
  126:     true = (ErrorLine =< EndLine),
  127:     String = lists:flatten(Module0:format_error(Desc)),
  128:     true = io_lib:printable_list(String).
  129: 
  130: iso88591(doc) -> ["Tests the support for ISO-8859-1 i.e Latin-1"];
  131: iso88591(suite) -> [];
  132: iso88591(Config) when is_list(Config) ->
  133:     ?line ok =
  134:      case catch begin
  135: 		   %% Some atom and variable names
  136: 		   V1s = [$Á,$á,$é,$ë],
  137: 		   V2s = [$N,$ä,$r],
  138: 		   A1s = [$h,$ä,$r],
  139: 		   A2s = [$ö,$r,$e],
  140: 		   %% Test parsing atom and variable characters.
  141: 		   {ok,Ts1,_} = erl_scan:string(V1s ++ " " ++ V2s ++
  142: 					       "\327" ++
  143: 					       A1s ++ " " ++ A2s),
  144: 		   V1s = atom_to_list(element(3, nth(1, Ts1))),
  145: 		   V2s = atom_to_list(element(3, nth(2, Ts1))),
  146: 		   A1s = atom_to_list(element(3, nth(4, Ts1))),
  147: 		   A2s = atom_to_list(element(3, nth(5, Ts1))),
  148: 		   %% Test printing atoms
  149: 		   A1s = flatten(print(element(3, nth(4, Ts1)))),
  150: 		   A2s = flatten(print(element(3, nth(5, Ts1)))),
  151: 		   %% Test parsing and printing strings.
  152: 		   S1 = V1s ++ "\327" ++ A1s ++ "\250" ++ A2s,
  153: 		   S1s = "\"" ++ S1 ++ "\"",
  154: 		   {ok,Ts2,_} = erl_scan:string(S1s),
  155: 		   S1 = element(3, nth(1, Ts2)),
  156: 		   S1s = flatten(print(element(3, nth(1, Ts2)))),
  157: 		   ok				%It all worked
  158: 	       end of
  159: 	{'EXIT',R} ->				%Something went wrong!
  160: 	     {error,R};
  161: 	ok -> ok				%Aok
  162:     end.
  163: 
  164: otp_7810(doc) ->
  165:     ["OTP-7810. White spaces, comments, and more.."];
  166: otp_7810(suite) ->
  167:     [];
  168: otp_7810(Config) when is_list(Config) ->
  169:     ?line ok = reserved_words(),
  170:     ?line ok = atoms(),
  171:     ?line ok = punctuations(),
  172:     ?line ok = comments(),
  173:     ?line ok = errors(),
  174:     ?line ok = integers(),
  175:     ?line ok = base_integers(),
  176:     ?line ok = floats(),
  177:     ?line ok = dots(),
  178:     ?line ok = chars(),
  179:     ?line ok = variables(),
  180:     ?line ok = eof(),
  181:     ?line ok = illegal(),
  182:     ?line ok = crashes(),
  183: 
  184:     ?line ok = options(),
  185:     ?line ok = token_info(),
  186:     ?line ok = column_errors(),
  187:     ?line ok = white_spaces(),
  188: 
  189:     ?line ok = unicode(),
  190: 
  191:     ?line ok = more_chars(),
  192:     ?line ok = more_options(),
  193:     ?line ok = attributes_info(),
  194:     ?line ok = set_attribute(),
  195: 
  196:     ok.
  197: 
  198: reserved_words() ->
  199:     L = ['after', 'begin', 'case', 'try', 'cond', 'catch',
  200:          'andalso', 'orelse', 'end', 'fun', 'if', 'let', 'of',
  201:          'receive', 'when', 'bnot', 'not', 'div',
  202:          'rem', 'band', 'and', 'bor', 'bxor', 'bsl', 'bsr',
  203:          'or', 'xor'],
  204:     [begin
  205:          ?line {RW, true} = {RW, erl_scan:reserved_word(RW)},
  206:          S = atom_to_list(RW),
  207:          Ts = [{RW,1}],
  208:          ?line test_string(S, Ts)
  209:      end || RW <- L],
  210:     ok.
  211: 
  212: 
  213: atoms() ->
  214:     ?line test_string("a
  215:                  b", [{atom,1,a},{atom,2,b}]),
  216:     ?line test_string("'a b'", [{atom,1,'a b'}]),
  217:     ?line test_string("a", [{atom,1,a}]),
  218:     ?line test_string("a@2", [{atom,1,a@2}]),
  219:     ?line test_string([39,65,200,39], [{atom,1,'AÈ'}]),
  220:     ?line test_string("ärlig östen", [{atom,1,ärlig},{atom,1,östen}]),
  221:     ?line {ok,[{atom,_,'$a'}],{1,6}} =
  222:         erl_scan:string("'$\\a'", {1,1}),
  223:     ?line test("'$\\a'"),
  224:     ok.
  225: 
  226: punctuations() ->
  227:     L = ["<<", "<-", "<=", "<", ">>", ">=", ">", "->", "--",
  228:          "-", "++", "+", "=:=", "=/=", "=<", "==", "=", "/=",
  229:          "/", "||", "|", ":-", "::", ":"],
  230:     %% One token at a time:
  231:     [begin
  232:          W = list_to_atom(S),
  233:          Ts = [{W,1}],
  234:          ?line test_string(S, Ts)
  235:      end || S <- L],
  236:     Three = ["/=:=", "<=:=", "==:=", ">=:="], % three tokens...
  237:     No = Three ++ L,
  238:     SL0 = [{S1++S2,{-length(S1),S1,S2}} ||
  239:               S1 <- L,
  240:               S2 <- L,
  241:               not lists:member(S1++S2, No)],
  242:     SL = family_list(SL0),
  243:     %% Two tokens. When there are several answers, the one with
  244:     %% the longest first token is chosen:
  245:     %% [the special case "=<<" is among the tested ones]
  246:     [begin
  247:          W1 = list_to_atom(S1),
  248:          W2 = list_to_atom(S2),
  249:          Ts = [{W1,1},{W2,1}],
  250:          ?line test_string(S, Ts)
  251:      end || {S,[{_,S1,S2}|_]}  <- SL],
  252: 
  253:     PTs1 = [{'!',1},{'(',1},{')',1},{',',1},{';',1},{'=',1},{'[',1},
  254:             {']',1},{'{',1},{'|',1},{'}',1}],
  255:     ?line test_string("!(),;=[]{|}", PTs1),
  256: 
  257:     PTs2 = [{'#',1},{'&',1},{'*',1},{'+',1},{'/',1},
  258:             {':',1},{'<',1},{'>',1},{'?',1},{'@',1},
  259:             {'\\',1},{'^',1},{'`',1},{'~',1}],
  260:     ?line test_string("#&*+/:<>?@\\^`~", PTs2),
  261: 
  262:     ?line test_string(".. ", [{'..',1}]),
  263:     ?line test("1 .. 2"),
  264:     ?line test_string("...", [{'...',1}]),
  265:     ok.
  266: 
  267: comments() ->
  268:     ?line test("a %%\n b"),
  269:     ?line {ok,[],1} = erl_scan:string("%"),
  270:     ?line test("a %%\n b"),
  271:     ?line {ok,[{atom,_,a},{atom,_,b}],{2,3}} =
  272:         erl_scan:string("a %%\n b",{1,1}),
  273:     ?line {ok,[{atom,_,a},{comment,_,"%%"},{atom,_,b}],{2,3}} =
  274:         erl_scan:string("a %%\n b",{1,1}, [return_comments]),
  275:     ?line {ok,[{atom,_,a},
  276:                {white_space,_," "},
  277:                {white_space,_,"\n "},
  278:                {atom,_,b}],
  279:            {2,3}} =
  280:         erl_scan:string("a %%\n b",{1,1},[return_white_spaces]),
  281:     ?line {ok,[{atom,_,a},
  282:                {white_space,_," "},
  283:                {comment,_,"%%"},
  284:                {white_space,_,"\n "},
  285:                {atom,_,b}],
  286:            {2,3}} = erl_scan:string("a %%\n b",{1,1},[return]),
  287:     ok.
  288: 
  289: errors() ->
  290:     ?line {error,{1,erl_scan,{string,$',"qa"}},1} = erl_scan:string("'qa"), %'
  291:     ?line {error,{1,erl_scan,{string,$","str"}},1} = %"
  292:         erl_scan:string("\"str"), %"
  293:     ?line {error,{1,erl_scan,char},1} = erl_scan:string("$"),
  294:     ?line test_string([34,65,200,34], [{string,1,"AÈ"}]),
  295:     ?line test_string("\\", [{'\\',1}]),
  296:     ?line {'EXIT',_} =
  297:         (catch {foo, erl_scan:string('$\\a', {1,1})}), % type error
  298:     ?line {'EXIT',_} =
  299:         (catch {foo, erl_scan:tokens([], '$\\a', {1,1})}), % type error
  300: 
  301:     ?line "{a,tuple}" = erl_scan:format_error({a,tuple}),
  302:     ok.
  303: 
  304: integers() ->
  305:     [begin
  306:          I = list_to_integer(S),
  307:          Ts = [{integer,1,I}],
  308:          ?line test_string(S, Ts)
  309:      end || S <- [[N] || N <- lists:seq($0, $9)] ++ ["2323","000"] ],
  310:     ok.
  311: 
  312: base_integers() ->
  313:     [begin
  314:          B = list_to_integer(BS),
  315:          I = erlang:list_to_integer(S, B),
  316:          Ts = [{integer,1,I}],
  317:          ?line test_string(BS++"#"++S, Ts)
  318:      end || {BS,S} <- [{"2","11"}, {"5","23234"}, {"12","05a"},
  319:                        {"16","abcdef"}, {"16","ABCDEF"}] ],
  320: 
  321:     ?line {error,{1,erl_scan,{base,1}},1} = erl_scan:string("1#000"),
  322: 
  323:     ?line test_string("12#bc", [{integer,1,11},{atom,1,c}]),
  324: 
  325:     [begin
  326:          Str = BS ++ "#" ++ S,
  327:          ?line {error,{1,erl_scan,{illegal,integer}},1} =
  328:              erl_scan:string(Str)
  329:      end || {BS,S} <- [{"3","3"},{"15","f"}, {"12","c"}] ],
  330: 
  331:     ?line {ok,[{integer,1,239},{'@',1}],1} = erl_scan:string("16#ef@"),
  332:     ?line {ok,[{integer,1,14},{atom,1,g@}],1} = erl_scan:string("16#eg@"),
  333: 
  334:     ok.
  335: 
  336: floats() ->
  337:     [begin
  338:          F = list_to_float(FS),
  339:          Ts = [{float,1,F}],
  340:          ?line test_string(FS, Ts)
  341:      end || FS <- ["1.0","001.17","3.31200","1.0e0","1.0E17",
  342:                    "34.21E-18", "17.0E+14"]],
  343:     ?line test_string("1.e2", [{integer,1,1},{'.',1},{atom,1,e2}]),
  344: 
  345:     ?line {error,{1,erl_scan,{illegal,float}},1} =
  346:         erl_scan:string("1.0e400"),
  347:     [begin
  348:          ?line {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(S)
  349:      end || S <- ["1.14Ea"]],
  350: 
  351:     ok.
  352: 
  353: dots() ->
  354:     Dot = [{".",    {ok,[{dot,1}],1}},
  355:            {". ",   {ok,[{dot,1}],1}},
  356:            {".\n",  {ok,[{dot,1}],2}},
  357:            {".%",   {ok,[{dot,1}],1}},
  358:            {".\210",{ok,[{dot,1}],1}},
  359:            {".% öh",{ok,[{dot,1}],1}},
  360:            {".%\n", {ok,[{dot,1}],2}},
  361:            {".$",   {error,{1,erl_scan,char},1}},
  362:            {".$\\", {error,{1,erl_scan,char},1}},
  363:            {".a",   {ok,[{'.',1},{atom,1,a}],1}}
  364:           ],
  365:     ?line [R = erl_scan:string(S) || {S, R} <- Dot],
  366: 
  367:     ?line {ok,[{dot,_}=T1],{1,2}} = erl_scan:string(".", {1,1}, text),
  368:     ?line [{column,1},{length,1},{line,1},{text,"."}] =
  369:         erl_scan:token_info(T1, [column, length, line, text]),
  370:     ?line {ok,[{dot,_}=T2],{1,3}} = erl_scan:string(".%", {1,1}, text),
  371:     ?line [{column,1},{length,1},{line,1},{text,"."}] =
  372:         erl_scan:token_info(T2, [column, length, line, text]),
  373:     ?line {ok,[{dot,_}=T3],{1,6}} =
  374:         erl_scan:string(".% öh", {1,1}, text),
  375:     ?line [{column,1},{length,1},{line,1},{text,"."}] =
  376:         erl_scan:token_info(T3, [column, length, line, text]),
  377:     ?line {error,{{1,2},erl_scan,char},{1,3}} =
  378:         erl_scan:string(".$", {1,1}),
  379:     ?line {error,{{1,2},erl_scan,char},{1,4}} =
  380:         erl_scan:string(".$\\", {1,1}),
  381: 
  382:     ?line test(". "),
  383:     ?line test(".  "),
  384:     ?line test(".\n"),
  385:     ?line test(".\n\n"),
  386:     ?line test(".\n\r"),
  387:     ?line test(".\n\n\n"),
  388:     ?line test(".\210"),
  389:     ?line test(".%\n"),
  390:     ?line test(".a"),
  391: 
  392:     ?line test("%. \n. "),
  393:     ?line {more,C} = erl_scan:tokens([], "%. ",{1,1}, return),
  394:     ?line {done,{ok,[{comment,_,"%. "},
  395:                      {white_space,_,"\n"},
  396:                      {dot,_}],
  397:                  {2,3}}, ""} =
  398:         erl_scan:tokens(C, "\n. ", {1,1}, return), % any loc, any options
  399: 
  400:     ?line [test_string(S, R) ||
  401:               {S, R} <- [{".$\n",   [{'.',1},{char,1,$\n}]},
  402:                          {"$\\\n",  [{char,1,$\n}]},
  403:                          {"'\\\n'", [{atom,1,'\n'}]},
  404:                          {"$\n",    [{char,1,$\n}]}] ],
  405:     ok.
  406: 
  407: chars() ->
  408:     [begin
  409:          L = lists:flatten(io_lib:format("$\\~.8b", [C])),
  410:          Ts = [{char,1,C}],
  411:          ?line test_string(L, Ts)
  412:      end || C <- lists:seq(0, 255)],
  413: 
  414:     %% Leading zeroes...
  415:     [begin
  416:          L = lists:flatten(io_lib:format("$\\~3.8.0b", [C])),
  417:          Ts = [{char,1,C}],
  418:          ?line test_string(L, Ts)
  419:      end || C <- lists:seq(0, 255)],
  420: 
  421:     %% $\^\n now increments the line...
  422:     [begin
  423:          L = "$\\^" ++ [C],
  424:          Ts = [{char,1,C band 2#11111}],
  425:          ?line test_string(L, Ts)
  426:      end || C <- lists:seq(0, 255)],
  427: 
  428:     [begin
  429:          L = "$\\" ++ [C],
  430:          Ts = [{char,1,V}],
  431:          ?line test_string(L, Ts)
  432:      end || {C,V} <- [{$n,$\n}, {$r,$\r}, {$t,$\t}, {$v,$\v},
  433:                       {$b,$\b}, {$f,$\f}, {$e,$\e}, {$s,$\s},
  434:                       {$d,$\d}]],
  435: 
  436:     EC = [$\n,$\r,$\t,$\v,$\b,$\f,$\e,$\s,$\d],
  437:     Ds = lists:seq($0, $9),
  438:     X = [$^,$n,$r,$t,$v,$b,$f,$e,$s,$d],
  439:     New = [${,$x],
  440:     No = EC ++ Ds ++ X ++ New,
  441:     [begin
  442:          L = "$\\" ++ [C],
  443:          Ts = [{char,1,C}],
  444:          ?line test_string(L, Ts)
  445:      end || C <- lists:seq(0, 255) -- No],
  446: 
  447:     [begin
  448:          L = "'$\\" ++ [C] ++ "'",
  449:          Ts = [{atom,1,list_to_atom("$"++[C])}],
  450:          ?line test_string(L, Ts)
  451:      end || C <- lists:seq(0, 255) -- No],
  452: 
  453:     ?line test_string("\"\\013a\\\n\"", [{string,1,"\va\n"}]),
  454: 
  455:     ?line test_string("'\n'", [{atom,1,'\n'}]),
  456:     ?line test_string("\"\n\a\"", [{string,1,"\na"}]),
  457: 
  458:     %% No escape
  459:     [begin
  460:          L = "$" ++ [C],
  461:          Ts = [{char,1,C}],
  462:          ?line test_string(L, Ts)
  463:      end || C <- lists:seq(0, 255) -- (No ++ [$\\])],
  464:     ?line test_string("$\n", [{char,1,$\n}]),
  465: 
  466:     ?line {error,{{1,1},erl_scan,char},{1,4}} =
  467:         erl_scan:string("$\\^",{1,1}),
  468:     ?line test_string("$\\\n", [{char,1,$\n}]),
  469:     %% Robert's scanner returns line 1:
  470:     ?line test_string("$\\\n", [{char,1,$\n}]),
  471:     ?line test_string("$\n\n", [{char,1,$\n}]),
  472:     ?line test("$\n\n"),
  473:     ok.
  474: 
  475: 
  476: variables() ->
  477:     ?line test_string("     \237_Aouåeiyäö", [{var,1,'_Aouåeiyäö'}]),
  478:     ?line test_string("A_b_c@", [{var,1,'A_b_c@'}]),
  479:     ?line test_string("V@2", [{var,1,'V@2'}]),
  480:     ?line test_string("ABDÀ", [{var,1,'ABDÀ'}]),
  481:     ?line test_string("Ärlig Östen", [{var,1,'Ärlig'},{var,1,'Östen'}]),
  482:     ok.
  483: 
  484: eof() ->
  485:     ?line {done,{eof,1},eof} = erl_scan:tokens([], eof, 1),
  486:     {more, C1} = erl_scan:tokens([],"    \n", 1),
  487:     ?line {done,{eof,2},eof} = erl_scan:tokens(C1, eof, 1),
  488:     {more, C2} = erl_scan:tokens([], "abra", 1),
  489:     %% An error before R13A.
  490:     %% ?line {done,Err={error,{1,erl_scan,scan},1},eof} =
  491:     ?line {done,{ok,[{atom,1,abra}],1},eof} =
  492:         erl_scan:tokens(C2, eof, 1),
  493: 
  494:     %% With column.
  495:     ?line {more, C3} = erl_scan:tokens([],"    \n",{1,1}),
  496:     ?line {done,{eof,{2,1}},eof} = erl_scan:tokens(C3, eof, 1),
  497:     {more, C4} = erl_scan:tokens([], "abra", {1,1}),
  498:     %% An error before R13A.
  499:     %% ?line {done,{error,{{1,1},erl_scan,scan},{1,5}},eof} =
  500:     ?line {done,{ok,[{atom,_,abra}],{1,5}},eof} =
  501:         erl_scan:tokens(C4, eof, 1),
  502: 
  503:     %% Robert's scanner returns "" as LeftoverChars;
  504:     %% the R12B scanner returns eof as LeftoverChars: (eof is correct)
  505:     ?line {more, C5} = erl_scan:tokens([], "a", 1),
  506:     %% An error before R13A.
  507:     %% ?line {done,{error,{1,erl_scan,scan},1},eof} =
  508:     ?line {done,{ok,[{atom,1,a}],1},eof} =
  509:         erl_scan:tokens(C5,eof,1),
  510: 
  511:     %% A dot followed by eof is special:
  512:     ?line {more, C} = erl_scan:tokens([], "a.", 1),
  513:     ?line {done,{ok,[{atom,1,a},{dot,1}],1},eof} = erl_scan:tokens(C,eof,1),
  514:     ?line {ok,[{atom,1,foo},{dot,1}],1} = erl_scan:string("foo."),
  515: 
  516:     ok.
  517: 
  518: illegal() ->
  519:     Atom = lists:duplicate(1000, $a),
  520:     ?line {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(Atom),
  521:     ?line {done,{error,{1,erl_scan,{illegal,atom}},1},". "} =
  522:         erl_scan:tokens([], Atom++". ", 1),
  523:     QAtom = "'" ++ Atom ++ "'",
  524:     ?line {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(QAtom),
  525:     ?line {done,{error,{1,erl_scan,{illegal,atom}},1},". "} =
  526:         erl_scan:tokens([], QAtom++". ", 1),
  527:     Var = lists:duplicate(1000, $A),
  528:     ?line {error,{1,erl_scan,{illegal,var}},1} = erl_scan:string(Var),
  529:     ?line {done,{error,{1,erl_scan,{illegal,var}},1},". "} =
  530:         erl_scan:tokens([], Var++". ", 1),
  531:     Float = "1" ++ lists:duplicate(400, $0) ++ ".0",
  532:     ?line {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(Float),
  533:     ?line {done,{error,{1,erl_scan,{illegal,float}},1},". "} =
  534:         erl_scan:tokens([], Float++". ", 1),
  535:     String = "\"43\\x{aaaaaa}34\"",
  536:     ?line {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string(String),
  537:     ?line {done,{error,{1,erl_scan,{illegal,character}},1},"34\". "} =
  538:         %% Would be nice if `34\"' were skipped...
  539:         %% Maybe, but then the LeftOverChars would not be the characters
  540:         %% immediately following the end location of the error.
  541:         erl_scan:tokens([], String++". ", 1),
  542: 
  543:     ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,1001}} =
  544:         erl_scan:string(Atom, {1,1}),
  545:     ?line {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1005}},". "} =
  546:         erl_scan:tokens([], "foo "++Atom++". ", {1,1}),
  547:     ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,1003}} =
  548:         erl_scan:string(QAtom, {1,1}),
  549:     ?line {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1007}},". "} =
  550:         erl_scan:tokens([], "foo "++QAtom++". ", {1,1}),
  551:     ?line {error,{{1,1},erl_scan,{illegal,var}},{1,1001}} =
  552:         erl_scan:string(Var, {1,1}),
  553:     ?line {done,{error,{{1,5},erl_scan,{illegal,var}},{1,1005}},". "} =
  554:         erl_scan:tokens([], "foo "++Var++". ", {1,1}),
  555:     ?line {error,{{1,1},erl_scan,{illegal,float}},{1,404}} =
  556:         erl_scan:string(Float, {1,1}),
  557:     ?line {done,{error,{{1,5},erl_scan,{illegal,float}},{1,408}},". "} =
  558:         erl_scan:tokens([], "foo "++Float++". ", {1,1}),
  559:     ?line {error,{{1,4},erl_scan,{illegal,character}},{1,14}} =
  560:         erl_scan:string(String, {1,1}),
  561:     ?line {done,{error,{{1,4},erl_scan,{illegal,character}},{1,14}},"34\". "} =
  562:         erl_scan:tokens([], String++". ", {1,1}),
  563:     ok.
  564: 
  565: crashes() ->
  566:     ?line {'EXIT',_} = (catch {foo, erl_scan:string([-1])}), % type error
  567:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("$"++[-1])}),
  568:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[-1])}),
  569:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[-1])}),
  570:     ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"],{1,1})}),
  571:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[-1,$"])}), %$"
  572:     ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"])}),
  573:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[-1])}),
  574:     ?line {'EXIT',_} =
  575:          (catch {foo, erl_scan:string("% foo"++[-1],{1,1})}),
  576: 
  577:     ?line {'EXIT',_} = (catch {foo, erl_scan:string([a])}), % type error
  578:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("$"++[a])}),
  579:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[a])}),
  580:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[a])}),
  581:     ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"],{1,1})}),
  582:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[a,$"])}), %$"
  583:     ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"])}),
  584:     ?line {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[a])}),
  585:     ?line {'EXIT',_} =
  586:          (catch {foo, erl_scan:string("% foo"++[a],{1,1})}),
  587: 
  588:     ?line {'EXIT',_} = (catch {foo, erl_scan:string([3.0])}), % type error
  589: 
  590:     ok.
  591: 
  592: options() ->
  593:     %% line and column are not options, but tested here
  594:     ?line {ok,[{atom,1,foo},{white_space,1," "},{comment,1,"% bar"}], 1} =
  595:         erl_scan:string("foo % bar", 1, return),
  596:     ?line {ok,[{atom,1,foo},{white_space,1," "}],1} =
  597:         erl_scan:string("foo % bar", 1, return_white_spaces),
  598:     ?line {ok,[{atom,1,foo},{comment,1,"% bar"}],1} =
  599:         erl_scan:string("foo % bar", 1, return_comments),
  600:     ?line {ok,[{atom,17,foo}],17} =
  601:         erl_scan:string("foo % bar", 17),
  602:     ?line {'EXIT',{function_clause,_}} =
  603:         (catch {foo,
  604:                 erl_scan:string("foo % bar", {a,1}, [])}), % type error
  605:     ?line {ok,[{atom,_,foo}],{17,18}} =
  606:         erl_scan:string("foo % bar", {17,9}, []),
  607:     ?line {'EXIT',{function_clause,_}} =
  608:         (catch {foo,
  609:                 erl_scan:string("foo % bar", {1,0}, [])}), % type error
  610:     ?line {ok,[{foo,1}],1} =
  611:         erl_scan:string("foo % bar",1, [{reserved_word_fun,
  612:                                          fun(W) -> W =:= foo end}]),
  613:     ?line {'EXIT',{badarg,_}} =
  614:         (catch {foo,
  615:                 erl_scan:string("foo % bar",1, % type error
  616:                                 [{reserved_word_fun,
  617:                                   fun(W,_) -> W =:= foo end}])}),
  618:     ok.
  619: 
  620: more_options() ->
  621:     ?line {ok,[{atom,A1,foo}],{19,20}} =
  622:         erl_scan:string("foo", {19,17},[]),
  623:     ?line [{column,17},{line,19}] = erl_scan:attributes_info(A1),
  624:     ?line {done,{ok,[{atom,A2,foo},{dot,_}],{19,22}},[]} =
  625:         erl_scan:tokens([], "foo. ", {19,17}, [bad_opt]), % type error
  626:     ?line [{column,17},{line,19}] = erl_scan:attributes_info(A2),
  627:     ?line {ok,[{atom,A3,foo}],{19,20}} =
  628:         erl_scan:string("foo", {19,17},[text]),
  629:     ?line [{column,17},{length,3},{line,19},{text,"foo"}] =
  630:         erl_scan:attributes_info(A3),
  631: 
  632:     ?line {ok,[{atom,A4,foo}],1} = erl_scan:string("foo", 1, [text]),
  633:     ?line [{length,3},{line,1},{text,"foo"}] = erl_scan:attributes_info(A4),
  634: 
  635:     ok.
  636: 
  637: token_info() ->
  638:     ?line {ok,[T1],_} = erl_scan:string("foo", {1,18}, [text]),
  639:     {'EXIT',{badarg,_}} =
  640:         (catch {foo, erl_scan:token_info(T1, foo)}), % type error
  641:     ?line {line,1} = erl_scan:token_info(T1, line),
  642:     ?line {column,18} = erl_scan:token_info(T1, column),
  643:     ?line {length,3} = erl_scan:token_info(T1, length),
  644:     ?line {text,"foo"} = erl_scan:token_info(T1, text),
  645:     ?line [{category,atom},{column,18},{length,3},{line,1},
  646:            {symbol,foo},{text,"foo"}] =
  647:         erl_scan:token_info(T1),
  648:     ?line [{length,3},{column,18}] =
  649:         erl_scan:token_info(T1, [length, column]),
  650:     ?line [{location,{1,18}}] =
  651:         erl_scan:token_info(T1, [location]),
  652:     ?line {category,atom} = erl_scan:token_info(T1, category),
  653:     ?line [{symbol,foo}] = erl_scan:token_info(T1, [symbol]),
  654: 
  655:     ?line {ok,[T2],_} = erl_scan:string("foo", 1, []),
  656:     ?line {line,1} = erl_scan:token_info(T2, line),
  657:     ?line undefined = erl_scan:token_info(T2, column),
  658:     ?line undefined = erl_scan:token_info(T2, length),
  659:     ?line undefined = erl_scan:token_info(T2, text),
  660:     ?line {location,1} = erl_scan:token_info(T2, location),
  661:     ?line [{category,atom},{line,1},{symbol,foo}] = erl_scan:token_info(T2),
  662:     ?line [{line,1}] = erl_scan:token_info(T2, [length, line]),
  663: 
  664:     ?line {ok,[T3],_} = erl_scan:string("=", 1, []),
  665:     ?line [{line,1}] = erl_scan:token_info(T3, [column, line]),
  666:     ?line {category,'='} = erl_scan:token_info(T3, category),
  667:     ?line [{symbol,'='}] = erl_scan:token_info(T3, [symbol]),
  668:     ok.
  669: 
  670: attributes_info() ->
  671:     ?line {'EXIT',_} =
  672:         (catch {foo,erl_scan:attributes_info(foo)}), % type error
  673:     ?line [{line,18}] = erl_scan:attributes_info(18),
  674:     ?line {location,19} = erl_scan:attributes_info(19, location),
  675:     ?line {ok,[{atom,A0,foo}],_} = erl_scan:string("foo", 19, [text]),
  676:     ?line {location,19} = erl_scan:attributes_info(A0, location),
  677: 
  678:     ?line {ok,[{atom,A3,foo}],_} = erl_scan:string("foo", {1,3}, [text]),
  679:     ?line {line,1} = erl_scan:attributes_info(A3, line),
  680:     ?line {column,3} = erl_scan:attributes_info(A3, column),
  681:     ?line {location,{1,3}} = erl_scan:attributes_info(A3, location),
  682:     ?line {text,"foo"} = erl_scan:attributes_info(A3, text),
  683: 
  684:     ?line {ok,[{atom,A4,foo}],_} = erl_scan:string("foo", 2, [text]),
  685:     ?line {line,2} = erl_scan:attributes_info(A4, line),
  686:     ?line undefined = erl_scan:attributes_info(A4, column),
  687:     ?line {location,2} = erl_scan:attributes_info(A4, location),
  688:     ?line {text,"foo"} = erl_scan:attributes_info(A4, text),
  689: 
  690:     ?line {ok,[{atom,A5,foo}],_} = erl_scan:string("foo", {1,3}, []),
  691:     ?line {line,1} = erl_scan:attributes_info(A5, line),
  692:     ?line {column,3} = erl_scan:attributes_info(A5, column),
  693:     ?line {location,{1,3}} = erl_scan:attributes_info(A5, location),
  694:     ?line undefined = erl_scan:attributes_info(A5, text),
  695: 
  696:     ?line undefined = erl_scan:attributes_info([], line), % type error
  697: 
  698:     ok.
  699: 
  700: set_attribute() ->
  701:     F = fun(Line) -> -Line end,
  702:     ?line -2 = erl_scan:set_attribute(line, 2, F),
  703:     ?line {ok,[{atom,A1,foo}],_} = erl_scan:string("foo", {9,17}),
  704:     ?line A2 = erl_scan:set_attribute(line, A1, F),
  705:     ?line {line,-9} = erl_scan:attributes_info(A2, line),
  706:     ?line {location,{-9,17}} = erl_scan:attributes_info(A2, location),
  707:     ?line [{line,-9},{column,17}] =
  708:         erl_scan:attributes_info(A2, [line,column,text]),
  709: 
  710:     F2 = fun(Line) -> {17,Line} end,
  711:     ?line Attr1 = erl_scan:set_attribute(line, 2, F2),
  712:     ?line {line,{17,2}} = erl_scan:attributes_info(Attr1, line),
  713:     ?line undefined = erl_scan:attributes_info(Attr1, column),
  714:     ?line {location,{17,2}} = % a bit mixed up
  715:         erl_scan:attributes_info(Attr1, location),
  716: 
  717:     ?line A3 = erl_scan:set_attribute(line, A1, F2),
  718:     ?line {line,{17,9}} = erl_scan:attributes_info(A3, line),
  719:     ?line {location,{{17,9},17}} = erl_scan:attributes_info(A3, location),
  720:     ?line [{line,{17,9}},{column,17}] =
  721:         erl_scan:attributes_info(A3, [line,column,text]),
  722: 
  723:     ?line {ok,[{atom,A4,foo}],_} = erl_scan:string("foo", {9,17}, [text]),
  724:     ?line A5 = erl_scan:set_attribute(line, A4, F),
  725:     ?line {line,-9} = erl_scan:attributes_info(A5, line),
  726:     ?line {location,{-9,17}} = erl_scan:attributes_info(A5, location),
  727:     ?line [{line,-9},{column,17},{text,"foo"}] =
  728:         erl_scan:attributes_info(A5, [line,column,text]),
  729: 
  730:     ?line {ok,[{atom,A6,foo}],_} = erl_scan:string("foo", 11, [text]),
  731:     ?line A7 = erl_scan:set_attribute(line, A6, F2),
  732:     ?line {line,{17,11}} = erl_scan:attributes_info(A7, line),
  733:     ?line {location,{17,11}} = % mixed up
  734:         erl_scan:attributes_info(A7, location),
  735:     ?line [{line,{17,11}},{text,"foo"}] =
  736:         erl_scan:attributes_info(A7, [line,column,text]),
  737: 
  738:     ?line {'EXIT',_} =
  739:         (catch {foo, erl_scan:set_attribute(line, [], F2)}), % type error
  740:     ?line {'EXIT',{badarg,_}} =
  741:         (catch {foo, erl_scan:set_attribute(column, [], F2)}), % type error
  742: 
  743:     %% OTP-9412
  744:     ?line 8 = erl_scan:set_attribute(line, [{line,{nos,'X',8}}],
  745:                                      fun({nos,_V,VL}) -> VL end),
  746:     ok.
  747: 
  748: column_errors() ->
  749:     ?line {error,{{1,1},erl_scan,{string,$',""}},{1,3}} = % $'
  750:         erl_scan:string("'\\",{1,1}),
  751:     ?line {error,{{1,1},erl_scan,{string,$",""}},{1,3}} = % $"
  752:         erl_scan:string("\"\\",{1,1}),
  753: 
  754:     ?line {error,{{1,1},erl_scan,{string,$',""}},{1,2}} =  % $'
  755:         erl_scan:string("'",{1,1}),
  756:     ?line {error,{{1,1},erl_scan,{string,$",""}},{1,2}} =  % $"
  757:         erl_scan:string("\"",{1,1}),
  758: 
  759:     ?line {error,{{1,1},erl_scan,char},{1,2}} =
  760:         erl_scan:string("$",{1,1}),
  761: 
  762:     ?line {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{1,20}} = %'
  763:         erl_scan:string(" '12345678901234567", {1,1}),
  764:     ?line {error,{{1,2},erl_scan,{string,$',"123456789012345 "}}, {1,20}} = %'
  765:         erl_scan:string(" '123456789012345\\s", {1,1}),
  766:     ?line {error,{{1,2},erl_scan,{string,$","1234567890123456"}},{1,20}} = %"
  767:         erl_scan:string(" \"12345678901234567", {1,1}),
  768:     ?line {error,{{1,2},erl_scan,{string,$","123456789012345 "}}, {1,20}} = %"
  769:         erl_scan:string(" \"123456789012345\\s", {1,1}),
  770:     ?line {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{2,1}} = %'
  771:         erl_scan:string(" '12345678901234567\n", {1,1}),
  772:     ok.
  773: 
  774: white_spaces() ->
  775:     ?line {ok,[{white_space,_,"\r"},
  776:                {white_space,_,"   "},
  777:                {atom,_,a},
  778:                {white_space,_,"\n"}],
  779:            _} = erl_scan:string("\r   a\n", {1,1}, return),
  780:     ?line test("\r   a\n"),
  781:     L = "{\"a\nb\", \"a\\nb\",\nabc\r,def}.\n\n",
  782:     ?line {ok,[{'{',_},
  783:                {string,_,"a\nb"},
  784:                {',',_},
  785:                {white_space,_," "},
  786:                {string,_,"a\nb"},
  787:                {',',_},
  788:                {white_space,_,"\n"},
  789:                {atom,_,abc},
  790:                {white_space,_,"\r"},
  791:                {',',_},
  792:                {atom,_,def},
  793:                {'}',_},
  794:                {dot,_},
  795:                {white_space,_,"\n"}],
  796:            _} = erl_scan:string(L, {1,1}, return),
  797:     ?line test(L),
  798:     ?line test("\"\n\"\n"),
  799:     ?line test("\n\r\n"),
  800:     ?line test("\n\r"),
  801:     ?line test("\r\n"),
  802:     ?line test("\n\f"),
  803:     ?line [test(lists:duplicate(N, $\t)) || N <- lists:seq(1, 20)],
  804:     ?line [test([$\n|lists:duplicate(N, $\t)]) || N <- lists:seq(1, 20)],
  805:     ?line [test(lists:duplicate(N, $\s)) || N <- lists:seq(1, 20)],
  806:     ?line [test([$\n|lists:duplicate(N, $\s)]) || N <- lists:seq(1, 20)],
  807:     ?line test("\v\f\n\v "),
  808:     ?line test("\n\e\n\b\f\n\da\n"),
  809:     ok.
  810: 
  811: unicode() ->
  812:     ?line {ok,[{char,1,83},{integer,1,45}],1} =
  813:         erl_scan:string("$\\12345"), % not unicode
  814: 
  815:     ?line {error,{1,erl_scan,{illegal,character}},1} =
  816:         erl_scan:string([1089]),
  817:     ?line {error,{{1,1},erl_scan,{illegal,character}},{1,2}} =
  818:         erl_scan:string([1089], {1,1}),
  819:     ?line {error,{1,erl_scan,{illegal,atom}},1} =
  820:         erl_scan:string("'a"++[1089]++"b'", 1),
  821:     ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,6}} =
  822:         erl_scan:string("'a"++[1089]++"b'", {1,1}),
  823:     ?line test("\"a"++[1089]++"b\""),
  824:     ?line {ok,[{char,1,1}],1} =
  825:         erl_scan:string([$$,$\\,$^,1089], 1),
  826: 
  827:     ?line {error,{1,erl_scan,Error},1} =
  828:         erl_scan:string("\"qa\x{aaa}", 1),
  829:     ?line "unterminated string starting with \"qa"++[2730]++"\"" =
  830:         erl_scan:format_error(Error),
  831:     ?line {error,{{1,1},erl_scan,_},{1,11}} =
  832:         erl_scan:string("\"qa\\x{aaa}",{1,1}),
  833:     ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} =
  834:         erl_scan:string("'qa\\x{aaa}'",{1,1}),
  835: 
  836:     ?line {ok,[{char,1,1089}],1} =
  837:         erl_scan:string([$$,1089], 1),
  838:     ?line {ok,[{char,1,1089}],1} =
  839:         erl_scan:string([$$,$\\,1089], 1),
  840: 
  841:     Qs = "$\\x{aaa}",
  842:     ?line {ok,[{char,1,$\x{aaa}}],1} =
  843:         erl_scan:string(Qs, 1),
  844:     ?line {ok,[Q2],{1,9}} =
  845:         erl_scan:string("$\\x{aaa}", {1,1}, [text]),
  846:     ?line [{category,char},{column,1},{length,8},
  847:            {line,1},{symbol,16#aaa},{text,Qs}] =
  848:         erl_scan:token_info(Q2),
  849: 
  850:     U1 = "\"\\x{aaa}\"",
  851:     {ok,
  852:      [{string,[{line,1},{column,1},{text,"\"\\x{aaa}\""}],[2730]}],
  853:      {1,10}} = erl_scan:string(U1, {1,1}, [text]),
  854:     {ok,[{string,1,[2730]}],1} = erl_scan:string(U1, 1),
  855: 
  856:     U2 = "\"\\x41\\x{fff}\\x42\"",
  857:     {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan:string(U2, 1),
  858: 
  859:     U3 = "\"a\n\\x{fff}\n\"",
  860:     {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan:string(U3, 1),
  861: 
  862:     U4 = "\"\\^\n\\x{aaa}\\^\n\"",
  863:     {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan:string(U4, 1),
  864: 
  865:     %% Keep these tests:
  866:     ?line test(Qs),
  867:     ?line test(U1),
  868:     ?line test(U2),
  869:     ?line test(U3),
  870:     ?line test(U4),
  871: 
  872:     Str1 = "\"ab" ++ [1089] ++ "cd\"",
  873:     {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan:string(Str1, 1),
  874:     {ok,[{string,{1,1},[$a,$b,1089,$c,$d]}],{1,8}} =
  875:         erl_scan:string(Str1, {1,1}),
  876:     ?line test(Str1),
  877:     Comment = "%% "++[1089],
  878:     {ok,[{comment,1,[$%,$%,$\s,1089]}],1} =
  879:         erl_scan:string(Comment, 1, [return]),
  880:     {ok,[{comment,{1,1},[$%,$%,$\s,1089]}],{1,5}} =
  881:         erl_scan:string(Comment, {1,1}, [return]),
  882:     ok.
  883: 
  884: more_chars() ->
  885:     %% Due to unicode, the syntax has been incompatibly augmented:
  886:     %% $\x{...}, $\xHH
  887: 
  888:     %% All kinds of tests...
  889:     ?line {ok,[{char,_,123}],{1,4}} =
  890:         erl_scan:string("$\\{",{1,1}),
  891:     ?line {more, C1} = erl_scan:tokens([], "$\\{", {1,1}),
  892:     ?line {done,{ok,[{char,_,123}],{1,4}},eof} =
  893:         erl_scan:tokens(C1, eof, 1),
  894:     ?line {ok,[{char,1,123},{atom,1,a},{'}',1}],1} =
  895:         erl_scan:string("$\\{a}"),
  896: 
  897:     ?line {error,{{1,1},erl_scan,char},{1,4}} =
  898:         erl_scan:string("$\\x", {1,1}),
  899:     ?line {error,{{1,1},erl_scan,char},{1,5}} =
  900:         erl_scan:string("$\\x{",{1,1}),
  901:     ?line {more, C3} = erl_scan:tokens([], "$\\x", {1,1}),
  902:     ?line {done,{error,{{1,1},erl_scan,char},{1,4}},eof} =
  903:         erl_scan:tokens(C3, eof, 1),
  904:     ?line {error,{{1,1},erl_scan,char},{1,5}} =
  905:         erl_scan:string("$\\x{",{1,1}),
  906:     ?line {more, C2} = erl_scan:tokens([], "$\\x{", {1,1}),
  907:     ?line {done,{error,{{1,1},erl_scan,char},{1,5}},eof} =
  908:         erl_scan:tokens(C2, eof, 1),
  909:     ?line {error,{1,erl_scan,{illegal,character}},1} =
  910:         erl_scan:string("$\\x{g}"),
  911:     ?line {error,{{1,1},erl_scan,{illegal,character}},{1,5}} =
  912:         erl_scan:string("$\\x{g}", {1,1}),
  913:     ?line {error,{{1,1},erl_scan,{illegal,character}},{1,6}} =
  914:         erl_scan:string("$\\x{}",{1,1}),
  915: 
  916:     ?line test("\"\\{0}\""),
  917:     ?line test("\"\\x{0}\""),
  918:     ?line test("\'\\{0}\'"),
  919:     ?line test("\'\\x{0}\'"),
  920: 
  921:     ?line {error,{{2,3},erl_scan,{illegal,character}},{2,6}} =
  922:         erl_scan:string("\"ab \n $\\x{g}\"",{1,1}),
  923:     ?line {error,{{2,3},erl_scan,{illegal,character}},{2,6}} =
  924:         erl_scan:string("\'ab \n $\\x{g}\'",{1,1}),
  925: 
  926:     ?line test("$\\{34}"),
  927:     ?line test("$\\x{34}"),
  928:     ?line test("$\\{377}"),
  929:     ?line test("$\\x{FF}"),
  930:     ?line test("$\\{400}"),
  931:     ?line test("$\\x{100}"),
  932:     ?line test("$\\x{10FFFF}"),
  933:     ?line test("$\\x{10ffff}"),
  934:     ?line test("\"$\n \\{1}\""),
  935:     ?line {error,{1,erl_scan,{illegal,character}},1} =
  936:         erl_scan:string("$\\x{110000}"),
  937:     ?line {error,{{1,1},erl_scan,{illegal,character}},{1,12}} =
  938:         erl_scan:string("$\\x{110000}", {1,1}),
  939: 
  940:     ?line {error,{{1,1},erl_scan,{illegal,character}},{1,4}} =
  941:         erl_scan:string("$\\xfg", {1,1}),
  942: 
  943:     ?line test("$\\xffg"),
  944: 
  945:     ?line {error,{{1,1},erl_scan,{illegal,character}},{1,4}} =
  946:         erl_scan:string("$\\xg", {1,1}),
  947:     ok.
  948: 
  949: otp_10302(doc) ->
  950:     "OTP-10302. Unicode characters scanner/parser.";
  951: otp_10302(suite) ->
  952:     [];
  953: otp_10302(Config) when is_list(Config) ->
  954:     %% From unicode():
  955:     {error,{1,erl_scan,{illegal,atom}},1} =
  956:         erl_scan:string("'a"++[1089]++"b'", 1),
  957:     {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} =
  958:         erl_scan:string("'qa\\x{aaa}'",{1,1}),
  959: 
  960:     {ok,[{char,1,1089}],1} = erl_scan:string([$$,1089], 1),
  961:     {ok,[{char,1,1089}],1} = erl_scan:string([$$,$\\,1089],1),
  962: 
  963:     Qs = "$\\x{aaa}",
  964:     {ok,[{char,1,2730}],1} = erl_scan:string(Qs,1),
  965:     {ok,[Q2],{1,9}} = erl_scan:string(Qs,{1,1},[text]),
  966:     [{category,char},{column,1},{length,8},
  967:      {line,1},{symbol,16#aaa},{text,Qs}] =
  968:         erl_scan:token_info(Q2),
  969: 
  970:     Tags = [category, column, length, line, symbol, text],
  971: 
  972:     U1 = "\"\\x{aaa}\"",
  973:     {ok,[T1],{1,10}} = erl_scan:string(U1, {1,1}, [text]),
  974:     [{category,string},{column,1},{length,9},{line,1},
  975:      {symbol,[16#aaa]},{text,U1}] = erl_scan:token_info(T1, Tags),
  976: 
  977:     U2 = "\"\\x41\\x{fff}\\x42\"",
  978:     {ok,[{string,1,[65,4095,66]}],1} = erl_scan:string(U2, 1),
  979: 
  980:     U3 = "\"a\n\\x{fff}\n\"",
  981:     {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan:string(U3, 1),
  982: 
  983:     U4 = "\"\\^\n\\x{aaa}\\^\n\"",
  984:     {ok,[{string,1,[10,2730,10]}],3} = erl_scan:string(U4, 1,[]),
  985: 
  986:     Str1 = "\"ab" ++ [1089] ++ "cd\"",
  987:     {ok,[{string,1,[97,98,1089,99,100]}],1} =
  988:         erl_scan:string(Str1,1),
  989:     {ok,[{string,{1,1},[97,98,1089,99,100]}],{1,8}} =
  990:         erl_scan:string(Str1, {1,1}),
  991: 
  992:     OK1 = 16#D800-1,
  993:     OK2 = 16#DFFF+1,
  994:     OK3 = 16#FFFE-1,
  995:     OK4 = 16#FFFF+1,
  996:     OKL = [OK1,OK2,OK3,OK4],
  997: 
  998:     Illegal1 = 16#D800,
  999:     Illegal2 = 16#DFFF,
 1000:     Illegal3 = 16#FFFE,
 1001:     Illegal4 = 16#FFFF,
 1002:     IllegalL = [Illegal1,Illegal2,Illegal3,Illegal4],
 1003: 
 1004:     [{ok,[{comment,1,[$%,$%,$\s,OK]}],1} =
 1005:          erl_scan:string("%% "++[OK], 1, [return]) ||
 1006:         OK <- OKL],
 1007:     {ok,[{comment,_,[$%,$%,$\s,OK1]}],{1,5}} =
 1008:         erl_scan:string("%% "++[OK1], {1,1}, [return]),
 1009:     [{error,{1,erl_scan,{illegal,character}},1} =
 1010:          erl_scan:string("%% "++[Illegal], 1, [return]) ||
 1011:         Illegal <- IllegalL],
 1012:     {error,{{1,1},erl_scan,{illegal,character}},{1,5}} =
 1013:         erl_scan:string("%% "++[Illegal1], {1,1}, [return]),
 1014: 
 1015:     [{ok,[],1} = erl_scan:string("%% "++[OK], 1, []) ||
 1016:         OK <- OKL],
 1017:     {ok,[],{1,5}} = erl_scan:string("%% "++[OK1], {1,1}, []),
 1018:     [{error,{1,erl_scan,{illegal,character}},1} =
 1019:          erl_scan:string("%% "++[Illegal], 1, []) ||
 1020:         Illegal <- IllegalL],
 1021:     {error,{{1,1},erl_scan,{illegal,character}},{1,5}} =
 1022:         erl_scan:string("%% "++[Illegal1], {1,1}, []),
 1023: 
 1024:     [{ok,[{string,{1,1},[OK]}],{1,4}} =
 1025:         erl_scan:string("\""++[OK]++"\"",{1,1}) ||
 1026:         OK <- OKL],
 1027:     [{error,{{1,2},erl_scan,{illegal,character}},{1,3}} =
 1028:          erl_scan:string("\""++[OK]++"\"",{1,1}) ||
 1029:         OK <- IllegalL],
 1030: 
 1031:     [{error,{{1,1},erl_scan,{illegal,character}},{1,2}} =
 1032:         erl_scan:string([Illegal],{1,1}) ||
 1033:         Illegal <- IllegalL],
 1034: 
 1035:     {ok,[{char,{1,1},OK1}],{1,3}} =
 1036:         erl_scan:string([$$,OK1],{1,1}),
 1037:     {error,{{1,1},erl_scan,{illegal,character}},{1,2}} =
 1038:         erl_scan:string([$$,Illegal1],{1,1}),
 1039: 
 1040:     {ok,[{char,{1,1},OK1}],{1,4}} =
 1041:         erl_scan:string([$$,$\\,OK1],{1,1}),
 1042:     {error,{{1,1},erl_scan,{illegal,character}},{1,4}} =
 1043:         erl_scan:string([$$,$\\,Illegal1],{1,1}),
 1044: 
 1045:     {ok,[{string,{1,1},[55295]}],{1,5}} =
 1046:         erl_scan:string("\"\\"++[OK1]++"\"",{1,1}),
 1047:     {error,{{1,2},erl_scan,{illegal,character}},{1,4}} =
 1048:         erl_scan:string("\"\\"++[Illegal1]++"\"",{1,1}),
 1049: 
 1050:     {ok,[{char,{1,1},OK1}],{1,10}} =
 1051:         erl_scan:string("$\\x{D7FF}",{1,1}),
 1052:     {error,{{1,1},erl_scan,{illegal,character}},{1,10}} =
 1053:         erl_scan:string("$\\x{D800}",{1,1}),
 1054: 
 1055:     %% Not erl_scan, but erl_parse.
 1056:     {integer,0,1} = erl_parse:abstract(1),
 1057:     Float = 3.14, {float,0,Float} = erl_parse:abstract(Float),
 1058:     {nil,0} = erl_parse:abstract([]),
 1059:     {bin,0,
 1060:      [{bin_element,0,{integer,0,1},default,default},
 1061:       {bin_element,0,{integer,0,2},default,default}]} =
 1062:         erl_parse:abstract(<<1,2>>),
 1063:     {cons,0,{tuple,0,[{atom,0,a}]},{atom,0,b}} =
 1064:         erl_parse:abstract([{a} | b]),
 1065:     {string,0,"str"} = erl_parse:abstract("str"),
 1066:     {cons,0,
 1067:      {integer,0,$a},
 1068:      {cons,0,{integer,0,1024},{string,0,"c"}}} =
 1069:         erl_parse:abstract("a"++[1024]++"c"),
 1070: 
 1071:     Line = 17,
 1072:     {integer,Line,1} = erl_parse:abstract(1, Line),
 1073:     Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Line),
 1074:     {nil,Line} = erl_parse:abstract([], Line),
 1075:     {bin,Line,
 1076:      [{bin_element,Line,{integer,Line,1},default,default},
 1077:       {bin_element,Line,{integer,Line,2},default,default}]} =
 1078:         erl_parse:abstract(<<1,2>>, Line),
 1079:     {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
 1080:         erl_parse:abstract([{a} | b], Line),
 1081:     {string,Line,"str"} = erl_parse:abstract("str", Line),
 1082:     {cons,Line,
 1083:      {integer,Line,$a},
 1084:      {cons,Line,{integer,Line,1024},{string,Line,"c"}}} =
 1085:         erl_parse:abstract("a"++[1024]++"c", Line),
 1086: 
 1087:     Opts1 = [{line,17}],
 1088:     {integer,Line,1} = erl_parse:abstract(1, Opts1),
 1089:     Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts1),
 1090:     {nil,Line} = erl_parse:abstract([], Opts1),
 1091:     {bin,Line,
 1092:      [{bin_element,Line,{integer,Line,1},default,default},
 1093:       {bin_element,Line,{integer,Line,2},default,default}]} =
 1094:         erl_parse:abstract(<<1,2>>, Opts1),
 1095:     {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
 1096:         erl_parse:abstract([{a} | b], Opts1),
 1097:     {string,Line,"str"} = erl_parse:abstract("str", Opts1),
 1098:     {cons,Line,
 1099:      {integer,Line,$a},
 1100:      {cons,Line,{integer,Line,1024},{string,Line,"c"}}} =
 1101:         erl_parse:abstract("a"++[1024]++"c", Opts1),
 1102: 
 1103:     [begin
 1104:          {integer,Line,1} = erl_parse:abstract(1, Opts2),
 1105:          Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts2),
 1106:          {nil,Line} = erl_parse:abstract([], Opts2),
 1107:          {bin,Line,
 1108:           [{bin_element,Line,{integer,Line,1},default,default},
 1109:            {bin_element,Line,{integer,Line,2},default,default}]} =
 1110:              erl_parse:abstract(<<1,2>>, Opts2),
 1111:          {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
 1112:              erl_parse:abstract([{a} | b], Opts2),
 1113:          {string,Line,"str"} = erl_parse:abstract("str", Opts2),
 1114:          {string,Line,[97,1024,99]} =
 1115:              erl_parse:abstract("a"++[1024]++"c", Opts2)
 1116:      end || Opts2 <- [[{encoding,unicode},{line,Line}],
 1117:                       [{encoding,utf8},{line,Line}]]],
 1118: 
 1119:     {cons,0,
 1120:      {integer,0,97},
 1121:      {cons,0,{integer,0,1024},{string,0,"c"}}} =
 1122:         erl_parse:abstract("a"++[1024]++"c", [{encoding,latin1}]),
 1123:     ok.
 1124: 
 1125: otp_10990(doc) ->
 1126:     "OTP-10990. Floating point number in input string.";
 1127: otp_10990(suite) ->
 1128:     [];
 1129: otp_10990(Config) when is_list(Config) ->
 1130:     {'EXIT',_} = (catch {foo, erl_scan:string([$",42.0,$"],1)}),
 1131:     ok.
 1132: 
 1133: otp_10992(doc) ->
 1134:     "OTP-10992. List of floats to abstract format.";
 1135: otp_10992(suite) ->
 1136:     [];
 1137: otp_10992(Config) when is_list(Config) ->
 1138:     {cons,0,{float,0,42.0},{nil,0}} =
 1139:         erl_parse:abstract([42.0], [{encoding,unicode}]),
 1140:     {cons,0,{float,0,42.0},{nil,0}} =
 1141:         erl_parse:abstract([42.0], [{encoding,utf8}]),
 1142:     {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} =
 1143:         erl_parse:abstract([$A,42.0], [{encoding,unicode}]),
 1144:     {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} =
 1145:         erl_parse:abstract([$A,42.0], [{encoding,utf8}]),
 1146:     ok.
 1147: 
 1148: test_string(String, Expected) ->
 1149:     {ok, Expected, _End} = erl_scan:string(String),
 1150:     test(String).
 1151: 
 1152: %% test_string(String, Expected, StartLocation, Options) ->
 1153: %%     {ok, Expected, _End} = erl_scan:string(String, StartLocation, Options),
 1154: %%     test(String).
 1155: 
 1156: %% There are no checks of the tags...
 1157: test(String) ->
 1158:     %% io:format("Testing `~ts'~n", [String]),
 1159:     [{Tokens, End},
 1160:      {Wtokens, Wend},
 1161:      {Ctokens, Cend},
 1162:      {CWtokens, CWend},
 1163:      {CWtokens2, _}] =
 1164:         [scan_string_with_column(String, X) ||
 1165:             X <- [[],
 1166:                   [return_white_spaces],
 1167:                   [return_comments],
 1168:                   [return],
 1169:                   [return]]], % for white space compaction test
 1170: 
 1171:     {end1,End,Wend} = {end1,Wend,End},
 1172:     {end2,Wend,Cend} = {end2,Cend,Wend},
 1173:     {end3,Cend,CWend} = {end3,CWend,Cend},
 1174: 
 1175:     %% Test that the tokens that are common to two token lists are identical.
 1176:     {none,Tokens} = {none, filter_tokens(CWtokens, [white_space,comment])},
 1177:     {comments,Ctokens} =
 1178:         {comments,filter_tokens(CWtokens, [white_space])},
 1179:     {white_spaces,Wtokens} =
 1180:         {white_spaces,filter_tokens(CWtokens, [comment])},
 1181: 
 1182:     %% Use token attributes to extract parts from the original string,
 1183:     %% and check that the parts are identical to the token strings.
 1184:     {Line,Column} = test_decorated_tokens(String, CWtokens),
 1185:     {deco,{Line,Column},End} = {deco,End,{Line,Column}},
 1186: 
 1187:     %% Almost the same again: concat texts to get the original:
 1188:     Text = get_text(CWtokens),
 1189:     {text,Text,String} = {text,String,Text},
 1190: 
 1191:     %% Test that white spaces occupy less heap than the worst case.
 1192:     ok = test_white_space_compaction(CWtokens, CWtokens2),
 1193: 
 1194:     %% Test that white newlines are always first in text:
 1195:     WhiteTokens = select_tokens(CWtokens, [white_space]),
 1196:     ok = newlines_first(WhiteTokens),
 1197: 
 1198:     %% Line attribute only:
 1199:     [Simple,Wsimple,Csimple,WCsimple] = Simples =
 1200:         [element(2, erl_scan:string(String, 1, Opts)) ||
 1201:             Opts <- [[],
 1202:                      [return_white_spaces],
 1203:                      [return_comments],
 1204:                      [return]]],
 1205:     {consistent,true} = {consistent,consistent_attributes(Simples)},
 1206:     {simple_wc,WCsimple} = {simple_wc,simplify(CWtokens)},
 1207:     {simple,Simple} = {simple,filter_tokens(WCsimple, [white_space,comment])},
 1208:     {simple_c,Csimple} = {simple_c,filter_tokens(WCsimple, [white_space])},
 1209:     {simple_w,Wsimple} = {simple_w,filter_tokens(WCsimple, [comment])},
 1210: 
 1211:     %% Line attribute only, with text:
 1212:     [SimpleTxt,WsimpleTxt,CsimpleTxt,WCsimpleTxt] = SimplesTxt =
 1213:         [element(2, erl_scan:string(String, 1, [text|Opts])) ||
 1214:             Opts <- [[],
 1215:                      [return_white_spaces],
 1216:                      [return_comments],
 1217:                      [return]]],
 1218:     TextTxt = get_text(WCsimpleTxt),
 1219:     {text_txt,TextTxt,String} = {text_txt,String,TextTxt},
 1220:     {consistent_txt,true} =
 1221:         {consistent_txt,consistent_attributes(SimplesTxt)},
 1222:     {simple_txt,SimpleTxt} =
 1223:         {simple_txt,filter_tokens(WCsimpleTxt, [white_space,comment])},
 1224:     {simple_c_txt,CsimpleTxt} =
 1225:         {simple_c_txt,filter_tokens(WCsimpleTxt, [white_space])},
 1226:     {simple_w_txt,WsimpleTxt} =
 1227:         {simple_w_txt,filter_tokens(WCsimpleTxt, [comment])},
 1228: 
 1229:     ok.
 1230: 
 1231: test_white_space_compaction(Tokens, Tokens2) when Tokens =:= Tokens2 ->
 1232:     [WS, WS2] = [select_tokens(Ts, [white_space]) || Ts <- [Tokens, Tokens2]],
 1233:     test_wsc(WS, WS2).
 1234: 
 1235: test_wsc([], []) ->
 1236:     ok;
 1237: test_wsc([Token|Tokens], [Token2|Tokens2]) ->
 1238:     [Text, Text2] = [Text ||
 1239:                         {text, Text} <-
 1240:                             [erl_scan:token_info(T, text) ||
 1241:                                 T <- [Token, Token2]]],
 1242:     Sz = erts_debug:size(Text),
 1243:     Sz2 = erts_debug:size({Text, Text2}),
 1244:     IsCompacted = Sz2 < 2*Sz+erts_debug:size({a,a}),
 1245:     ToBeCompacted = is_compacted(Text),
 1246:     if
 1247:         IsCompacted =:= ToBeCompacted ->
 1248:             test_wsc(Tokens, Tokens2);
 1249:         true ->
 1250:             {compaction_error, Token}
 1251:     end.
 1252: 
 1253: is_compacted("\r") ->
 1254:     true;
 1255: is_compacted("\n\r") ->
 1256:     true;
 1257: is_compacted("\n\f") ->
 1258:     true;
 1259: is_compacted([$\n|String]) ->
 1260:       all_spaces(String)
 1261:     orelse
 1262:       all_tabs(String);
 1263: is_compacted(String) ->
 1264:       all_spaces(String)
 1265:     orelse
 1266:       all_tabs(String).
 1267: 
 1268: all_spaces(L) ->
 1269:     all_same(L, $\s).
 1270: 
 1271: all_tabs(L) ->
 1272:     all_same(L, $\t).
 1273: 
 1274: all_same(L, Char) ->
 1275:     lists:all(fun(C) -> C =:= Char end, L).
 1276: 
 1277: newlines_first([]) ->
 1278:     ok;
 1279: newlines_first([Token|Tokens]) ->
 1280:     {text,Text} = erl_scan:token_info(Token, text),
 1281:     Nnls = length([C || C <- Text, C =:= $\n]),
 1282:     OK = case Text of
 1283:              [$\n|_] ->
 1284:                  Nnls =:= 1;
 1285:              _ ->
 1286:                  Nnls =:= 0
 1287:          end,
 1288:     if
 1289:         OK -> newlines_first(Tokens);
 1290:         true -> OK
 1291:     end.
 1292: 
 1293: filter_tokens(Tokens, Tags) ->
 1294:     lists:filter(fun(T) -> not lists:member(element(1, T), Tags) end, Tokens).
 1295: 
 1296: select_tokens(Tokens, Tags) ->
 1297:     lists:filter(fun(T) -> lists:member(element(1, T), Tags) end, Tokens).
 1298: 
 1299: simplify([Token|Tokens]) ->
 1300:     {line,Line} = erl_scan:token_info(Token, line),
 1301:     [setelement(2, Token, Line) | simplify(Tokens)];
 1302: simplify([]) ->
 1303:     [].
 1304: 
 1305: get_text(Tokens) ->
 1306:     lists:flatten(
 1307:       [T ||
 1308:           Token <- Tokens,
 1309:           ({text,T} = erl_scan:token_info(Token, text)) =/= []]).
 1310: 
 1311: test_decorated_tokens(String, Tokens) ->
 1312:     ToksAttrs = token_attrs(Tokens),
 1313:     test_strings(ToksAttrs, String, 1, 1).
 1314: 
 1315: token_attrs(Tokens) ->
 1316:     [{L,C,Len,T} ||
 1317:         Token <- Tokens,
 1318:         ([{line,L},{column,C},{length,Len},{text,T}] =
 1319:          erl_scan:token_info(Token, [line,column,length,text])) =/= []].
 1320: 
 1321: test_strings([], _S, Line, Column) ->
 1322:     {Line,Column};
 1323: test_strings([{L,C,Len,T}=Attr|Attrs], String0, Line0, Column0) ->
 1324:     {String1, Column1} = skip_newlines(String0, L, Line0, Column0),
 1325:     String = skip_chars(String1, C-Column1),
 1326:     {Str,Rest} = lists:split(Len, String),
 1327:     if
 1328:         Str =:= T ->
 1329:             {Line,Column} = string_newlines(T, L, C),
 1330:             test_strings(Attrs, Rest, Line, Column);
 1331:         true ->
 1332:             {token_error, Attr, Str}
 1333:     end.
 1334: 
 1335: skip_newlines(String, Line, Line, Column) ->
 1336:     {String, Column};
 1337: skip_newlines([$\n|String], L, Line, _Column) ->
 1338:     skip_newlines(String, L, Line+1, 1);
 1339: skip_newlines([_|String], L, Line, Column) ->
 1340:     skip_newlines(String, L, Line, Column+1).
 1341: 
 1342: skip_chars(String, 0) ->
 1343:     String;
 1344: skip_chars([_|String], N) ->
 1345:     skip_chars(String, N-1).
 1346: 
 1347: string_newlines([$\n|String], Line, _Column) ->
 1348:     string_newlines(String, Line+1, 1);
 1349: string_newlines([], Line, Column) ->
 1350:     {Line, Column};
 1351: string_newlines([_|String], Line, Column) ->
 1352:     string_newlines(String, Line, Column+1).
 1353: 
 1354: scan_string_with_column(String, Options0) ->
 1355:     Options = [text | Options0],
 1356:     StartLoc = {1, 1},
 1357:     {ok, Ts1, End1} = erl_scan:string(String, StartLoc, Options),
 1358:     TString = String ++ ". ",
 1359:     {ok,Ts2,End2} = scan_tokens(TString, Options, [], StartLoc),
 1360:     {ok, Ts3, End3} =
 1361:         scan_tokens_1({more, []}, TString, Options, [], StartLoc),
 1362:     {end_2,End2,End3} = {end_2,End3,End2},
 1363:     {EndLine1,EndColumn1} = End1,
 1364:     End2 = {EndLine1,EndColumn1+2},
 1365:     {ts_1,Ts2,Ts3} = {ts_1,Ts3,Ts2},
 1366:     Ts2 = Ts1 ++ [lists:last(Ts2)],
 1367: 
 1368:     %% Attributes are keylists, but have no text.
 1369:     {ok, Ts7, End7} = erl_scan:string(String, {1,1}, Options),
 1370:     {ok, Ts8, End8} = scan_tokens(TString, Options, [], {1,1}),
 1371:     {end1, End1} = {end1, End7},
 1372:     {end2, End2} = {end2, End8},
 1373:     Ts8 = Ts7 ++ [lists:last(Ts8)],
 1374:     {cons,true} = {cons,consistent_attributes([Ts1,Ts2,Ts3,Ts7,Ts8])},
 1375: 
 1376:     {Ts1, End1}.
 1377: 
 1378: scan_tokens(String, Options, Rs, Location) ->
 1379:     case erl_scan:tokens([], String, Location, Options) of
 1380:         {done, {ok,Ts,End}, ""} ->
 1381:             {ok, lists:append(lists:reverse([Ts|Rs])), End};
 1382:         {done, {ok,Ts,End}, Rest} ->
 1383:             scan_tokens(Rest, Options, [Ts|Rs], End)
 1384:     end.
 1385: 
 1386: scan_tokens_1({done, {ok,Ts,End}, ""}, "", _Options, Rs, _Location) ->
 1387:     {ok,lists:append(lists:reverse([Ts|Rs])),End};
 1388: scan_tokens_1({done, {ok,Ts,End}, Rest}, Cs, Options, Rs, _Location) ->
 1389:     scan_tokens_1({more,[]}, Rest++Cs, Options, [Ts|Rs], End);
 1390: scan_tokens_1({more, Cont}, [C | Cs], Options, Rs, Loc) ->
 1391:     R = erl_scan:tokens(Cont, [C], Loc, Options),
 1392:     scan_tokens_1(R, Cs, Options, Rs, Loc).
 1393: 
 1394: consistent_attributes([]) ->
 1395:     true;
 1396: consistent_attributes([Ts | TsL]) ->
 1397:     L = [T || T <- Ts, is_integer(element(2, T))],
 1398:     case L of
 1399:         [] ->
 1400:             TagsL = [[Tag || {Tag,_} <-
 1401:                                  erl_scan:attributes_info(element(2, T))] ||
 1402:                         T <- Ts],
 1403:             case lists:usort(TagsL) of
 1404:                 [_] ->
 1405:                     consistent_attributes(TsL);
 1406:                 [] when Ts =:= [] ->
 1407:                     consistent_attributes(TsL);
 1408:                 _ ->
 1409:                     Ts
 1410:             end;
 1411:         Ts ->
 1412:             consistent_attributes(TsL);
 1413:         _ ->
 1414:             Ts
 1415:     end.
 1416: 
 1417: family_list(L) ->
 1418:     sofs:to_external(family(L)).
 1419: 
 1420: family(L) ->
 1421:     sofs:relation_to_family(sofs:relation(L)).