diff options
Diffstat (limited to 'lib')
350 files changed, 16504 insertions, 6482 deletions
diff --git a/lib/Makefile b/lib/Makefile index 6a12334d8d..9ddf3a0544 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2012. All Rights Reserved. +# Copyright Ericsson AB 1996-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/appmon/doc/src/notes.xml b/lib/appmon/doc/src/notes.xml index 52d7e1586d..9987ca9278 100644 --- a/lib/appmon/doc/src/notes.xml +++ b/lib/appmon/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/asn1/doc/src/notes.xml b/lib/asn1/doc/src/notes.xml index 5e21b926a8..4d4600b3ab 100644 --- a/lib/asn1/doc/src/notes.xml +++ b/lib/asn1/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,6 +31,22 @@ <p>This document describes the changes made to the asn1 application.</p> +<section><title>Asn1 2.0.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed broken table constraints within a SET OF or + SEQUENCE OF for the BER backend.</p> + <p> + Own Id: OTP-10853 Aux Id: seq12245 </p> + </item> + </list> + </section> + +</section> + <section><title>Asn1 2.0</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl index dd77085c39..e1911d8170 100644 --- a/lib/asn1/src/asn1ct_check.erl +++ b/lib/asn1/src/asn1ct_check.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1263,13 +1263,13 @@ check_object_list(S,ClassRef,[ObjOrSet|Objs],Acc) -> check_object_list(S,ClassRef,Objs,[{{no_mod,no_name},Def}|Acc]); {'ObjectSetFromObjects',Os,FieldName} when is_tuple(Os) -> NewSet = - check_ObjectSetFromObjects(S,element(size(Os),Os), + check_ObjectSetFromObjects(S, element(tuple_size(Os), Os), FieldName,[]), check_object_list(S,ClassRef,Objs,NewSet++Acc); {{'ObjectSetFromObjects',Os,FieldName},InterSection} when is_tuple(Os) -> NewSet = - check_ObjectSetFromObjects(S, element(size(Os),Os), + check_ObjectSetFromObjects(S, element(tuple_size(Os), Os), FieldName,InterSection), check_object_list(S,ClassRef,Objs,NewSet++Acc); Other -> @@ -1570,7 +1570,7 @@ gen_incl_set(S,Fields,#typedef{typespec=#type{def=Eref}}) gen_incl_set(S,Fields,CDef); gen_incl_set(S,Fields,ClassDef) -> case catch get_unique_fieldname(S,ClassDef) of - Tuple when is_tuple(Tuple), size(Tuple) =:= 3 -> + Tuple when tuple_size(Tuple) =:= 3 -> false; _ -> gen_incl_set1(S,Fields, @@ -1589,7 +1589,7 @@ gen_incl_set1(_,['EXTENSIONMARK'],_) -> gen_incl_set1(_,['EXTENSIONMARK'|_],_) -> true; gen_incl_set1(S,[Object|Rest],CFields)-> - Fields = element(size(Object),Object), + Fields = element(tuple_size(Object), Object), case gen_incl1(S,Fields,CFields) of true -> true; @@ -3028,7 +3028,7 @@ is_record_normalized(S,Name,V = #'Externalvaluereference'{},NumComps) -> _ -> false end; is_record_normalized(_S,Name,Value,NumComps) when is_tuple(Value) -> - (size(Value) =:= (NumComps + 1)) andalso (element(1,Value)=:=Name); + (tuple_size(Value) =:= (NumComps + 1)) andalso (element(1, Value) =:= Name); is_record_normalized(_,_,_,_) -> false. @@ -3720,7 +3720,7 @@ maybe_open_type(S,ClassSpec=#objectclass{fields=Fs}, {typefieldreference,_} -> case {catch get_unique_fieldname(S,#classdef{typespec=ClassSpec}), asn1ct_gen:get_constraint(Constr,componentrelation)}of - {Tuple,_} when is_tuple(Tuple), size(Tuple) =:= 3 -> + {Tuple,_} when tuple_size(Tuple) =:= 3 -> OCFT#'ObjectClassFieldType'{fieldname=FieldNames, type='ASN1_OPEN_TYPE'}; {_,no} -> @@ -4167,7 +4167,7 @@ check_constraint(S,Ext) when is_record(Ext,'Externaltypereference') -> check_constraint(S,{'SizeConstraint',{Lb,Ub}}) - when is_list(Lb);is_tuple(Lb),size(Lb)==2 -> + when is_list(Lb); tuple_size(Lb) =:= 2 -> NewLb = range_check(resolv_tuple_or_list(S,Lb)), NewUb = range_check(resolv_tuple_or_list(S,Ub)), {'SizeConstraint',{NewLb,NewUb}}; @@ -5217,7 +5217,7 @@ imported1(_Name,[]) -> check_integer(_S,[],_C) -> []; check_integer(S,NamedNumberList,_C) -> - case [X||X<-NamedNumberList,is_tuple(X),size(X)=:=2] of + case [X || X <- NamedNumberList, tuple_size(X) =:= 2] of NamedNumberList -> %% An already checked integer with NamedNumberList NamedNumberList; diff --git a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl index e82212f0d8..341a04761b 100644 --- a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl +++ b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl @@ -528,14 +528,7 @@ gen_decode_sof(Erules,TypeName,_InnerTypeName,D) when is_record(D,type) -> Atom when is_atom(Atom) -> Atom; _ -> TypeNameSuffix end, -%% fix me - ObjFun = - case D#type.tablecinf of - [{objfun,_}|_R] -> - ", ObjFun"; - _ -> - [] - end, + ObjFun = false, gen_dec_line(Erules,TypeName,ContName,[],Cont,mandatory,ObjFun), %% gen_dec_line_sof(Erules,Typename,ContName,Cont,ObjFun), emit([" || ",{curr,v}," <- ",{curr,tlv},"].",nl,nl,nl]). diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index ebc52df1d9..76c4182160 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -657,9 +657,13 @@ gen_check_sof(Name,SOF,Type) -> end, emit({" ",{asis,NewName},"(DVs,Vs).",nl,nl}). +gen_check_sequence(Name, []) -> + emit([{asis,ensure_atom(Name)},"(_,_) ->",nl, + " throw(badval).",nl,nl]); gen_check_sequence(Name,Components) -> emit([{asis,ensure_atom(Name)},"(DefaultValue,Value) ->",nl]), gen_check_sequence(Name,Components,1). + gen_check_sequence(Name,[#'ComponentType'{name=N,typespec=Type}|Cs],Num) -> InnerType = get_inner(Type#type.def), NthDefV = ["element(",Num+1,",DefaultValue)"], @@ -671,9 +675,7 @@ gen_check_sequence(Name,[#'ComponentType'{name=N,typespec=Type}|Cs],Num) -> _ -> emit({",",nl}), gen_check_sequence(Name,Cs,Num+1) - end; -gen_check_sequence(_,[],_) -> - ok. + end. gen_check_choice(Name,CList=[#'ComponentType'{}|_Cs]) -> emit([{asis,ensure_atom(Name)},"({Id,DefaultValue},{Id,Value}) ->",nl]), diff --git a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl index f3a2486565..de0adef2b2 100644 --- a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl +++ b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl @@ -1162,7 +1162,7 @@ gen_objset_enc(_,ObjSetName,_UniqueName,['EXTENSIONMARK'],_ClName, emit({"'getenc_",ObjSetName,"'(_, _) ->",nl}), emit({indent(3),"fun(_, Val, _RestPrimFieldName) ->",nl}), emit({indent(6),"Len = case Val of",nl,indent(9), - "Bin when is_binary(Bin) -> size(Bin);",nl,indent(9), + "Bin when is_binary(Bin) -> byte_size(Bin);",nl,indent(9), "_ -> length(Val)",nl,indent(6),"end,"}), emit({indent(6),"{Val,Len}",nl}), emit({indent(3),"end.",nl,nl}), @@ -1270,7 +1270,7 @@ gen_inlined_enc_funs1(Fields,[{typefield,Name,_}|Rest],ObjSetName, %% treatment. emit([";",nl,indent(9),{asis,Name}," ->",nl]), emit([indent(12),"Len = case Val of",nl, - indent(15),"Bin when is_binary(Bin) -> size(Bin);",nl, + indent(15),"Bin when is_binary(Bin) -> byte_size(Bin);",nl, indent(15),"_ -> length(Val)",nl,indent(12),"end,",nl, indent(12),"{Val,Len}"]), {Acc,0} @@ -1449,7 +1449,7 @@ gen_inlined_dec_funs(Fields,[{typefield,Name,Prop}|Rest], nl,indent(6),"case Type of",nl, indent(9),{asis,Name}," ->",nl, indent(12),"Len = case Bytes of",nl, - indent(15),"B when is_binary(B) -> size(B);",nl, + indent(15),"B when is_binary(B) -> byte_size(B);",nl, indent(15),"_ -> length(Bytes)",nl, indent(12),"end,",nl, indent(12),"{Bytes,[],Len}"]), diff --git a/lib/asn1/src/asn1ct_gen_per.erl b/lib/asn1/src/asn1ct_gen_per.erl index 0d6620667f..fac233532b 100644 --- a/lib/asn1/src/asn1ct_gen_per.erl +++ b/lib/asn1/src/asn1ct_gen_per.erl @@ -1174,12 +1174,12 @@ gen_dec_imm_1('UTF8String', _Constraint, Aligned) -> asn1ct_imm:per_dec_restricted_string(Aligned); gen_dec_imm_1('REAL', _Constraint, Aligned) -> asn1ct_imm:per_dec_real(Aligned); -gen_dec_imm_1(#'ObjectClassFieldType'{}=TypeName, Constraint, Aligned) -> +gen_dec_imm_1(#'ObjectClassFieldType'{}=TypeName, _Constraint, Aligned) -> case asn1ct_gen:get_inner(TypeName) of - {fixedtypevaluefield,_,InnerType} -> - gen_dec_imm_1(InnerType, Constraint, Aligned); - T -> - gen_dec_imm_1(T, Constraint, Aligned) + {fixedtypevaluefield,_,#type{def=InnerType,constraint=C}} -> + gen_dec_imm_1(InnerType, C, Aligned); + #type{def=T,constraint=C} -> + gen_dec_imm_1(T, C, Aligned) end. gen_dec_bit_string(F, Imm) -> diff --git a/lib/asn1/src/asn1ct_gen_per_rt2ct.erl b/lib/asn1/src/asn1ct_gen_per_rt2ct.erl index 5a409295fb..81d8cdcae6 100644 --- a/lib/asn1/src/asn1ct_gen_per_rt2ct.erl +++ b/lib/asn1/src/asn1ct_gen_per_rt2ct.erl @@ -610,9 +610,9 @@ gen_encode_objectfields(Erules,ClassName,[{typefield,Name,OptOrMand}|Rest], emit([" if",nl, " is_list(Val) ->",nl, " NewVal = list_to_binary(Val),",nl, - " [20,size(NewVal),NewVal];",nl, + " [20,byte_size(NewVal),NewVal];",nl, " is_binary(Val) ->",nl, - " [20,size(Val),Val]",nl, + " [20,byte_size(Val),Val]",nl, " end"]), []; {false,{'DEFAULT',DefaultType}} -> @@ -989,7 +989,7 @@ gen_objset_enc(_Erule,ObjSetName,_UniqueName,['EXTENSIONMARK'],_ClName, emit({indent(9),"is_list(Val) -> list_to_binary(Val);",nl}), emit({indent(9),"true -> Val",nl}), emit({indent(6),"end,",nl}), - emit({indent(6),"Size = size(BinVal),",nl}), + emit({indent(6),"Size = byte_size(BinVal),",nl}), emit({indent(6),"if",nl}), emit({indent(9),"Size < 256 ->",nl}), emit({indent(12),"[20,Size,BinVal];",nl}), diff --git a/lib/asn1/src/asn1rtt_ber.erl b/lib/asn1/src/asn1rtt_ber.erl index 88292aca99..5fbf116747 100644 --- a/lib/asn1/src/asn1rtt_ber.erl +++ b/lib/asn1/src/asn1rtt_ber.erl @@ -868,7 +868,7 @@ remove_unused_then_dotag(TagIn,Unused,BinBits) -> encode_tags(TagIn, <<0>>, 1); 0 -> Bin = <<Unused,BinBits/binary>>, - encode_tags(TagIn,Bin,size(Bin)); + encode_tags(TagIn, Bin, byte_size(Bin)); Num -> N = byte_size(BinBits)-1, <<BBits:N/binary,LastByte>> = BinBits, diff --git a/lib/asn1/src/asn1rtt_per.erl b/lib/asn1/src/asn1rtt_per.erl index d02f4f548e..84ff809912 100644 --- a/lib/asn1/src/asn1rtt_per.erl +++ b/lib/asn1/src/asn1rtt_per.erl @@ -613,11 +613,11 @@ bit_string_trailing_zeros1(BitList,Lb,Ub) -> encode_bin_bit_string(C, {Unused,BinBits}, _NamedBitList) when is_integer(C),C=<16 -> range_check(C, bit_size(BinBits) - Unused), - [45,C,size(BinBits),BinBits]; + [45,C,byte_size(BinBits),BinBits]; encode_bin_bit_string(C, {Unused,BinBits}, _NamedBitList) when is_integer(C), C =< 255 -> range_check(C, bit_size(BinBits) - Unused), - [2,45,C,size(BinBits),BinBits]; + [2,45,C,byte_size(BinBits),BinBits]; encode_bin_bit_string(C, {Unused,BinBits}, _NamedBitList) when is_integer(C), C =< 65535 -> range_check(C, bit_size(BinBits) - Unused), diff --git a/lib/asn1/src/asn1rtt_real_common.erl b/lib/asn1/src/asn1rtt_real_common.erl index 540f0d60a5..22a1f4c4dd 100644 --- a/lib/asn1/src/asn1rtt_real_common.erl +++ b/lib/asn1/src/asn1rtt_real_common.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012. All Rights Reserved. +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -88,7 +88,7 @@ encode_real(_C, {Mantissa, Base, Exponent}) when Base =:= 2 -> end, %% ok = io:format("SignBitMask: ~w~n",[SignBitMask]), SFactor = 0, - OctExpLen = size(OctExp), + OctExpLen = byte_size(OctExp), if OctExpLen > 255 -> exit({error,{asn1, {to_big_exp_in_encode_real, OctExpLen}}}); true -> true %% make real assert later.. diff --git a/lib/asn1/test/Makefile b/lib/asn1/test/Makefile index 1fa495d8f1..6e6ab4639c 100644 --- a/lib/asn1/test/Makefile +++ b/lib/asn1/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -109,6 +109,7 @@ MODULES= \ test_modified_x420 \ testX420 \ test_x691 \ + testWSParamClass \ asn1_test_lib \ asn1_app_test \ asn1_appup_test \ diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index be9b82cddf..62418e554e 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -86,6 +86,7 @@ groups() -> testInvokeMod, per, ber_other, + der, h323test, per_GeneralString]}, testChoPrim, @@ -166,13 +167,13 @@ groups() -> testINSTANCE_OF, testTCAP, test_ParamTypeInfObj, - test_WS_ParamClass, test_Defed_ObjectIdentifier, testSelectionType, testSSLspecs, testNortel, - % Uses 'PKCS7' - {group, [], [test_modified_x420, + % Uses 'PKCS7', 'InformationFramework' + {group, [], [test_WS_ParamClass, + test_modified_x420, testX420]}, testTcapsystem, testNBAPsystem, @@ -200,8 +201,6 @@ parallel(Options) -> %%------------------------------------------------------------------------------ init_per_suite(Config) -> - PrivDir = ?config(priv_dir, Config), - true = code:add_patha(PrivDir), Config. end_per_suite(_Config) -> @@ -214,7 +213,7 @@ end_per_group(_GroupName, Config) -> Config. init_per_testcase(Func, Config) -> - CaseDir = filename:join([?config(priv_dir, Config), ?MODULE, Func]), + CaseDir = filename:join(?config(priv_dir, Config), Func), ok = filelib:ensure_dir(filename:join([CaseDir, dummy_file])), true = code:add_patha(CaseDir), @@ -351,7 +350,7 @@ testPrimStrings(Config, Rule, Opts) -> asn1_test_lib:compile_all(["PrimStrings", "BitStr"], Config, [compact_bit_string,Rule|Opts]), testPrimStrings:bit_string(Rule), - ?only_ber(testPrimStrings:more_strings(Rule)). + testPrimStrings:more_strings(Rule). testPrimStrings_cases(Rule) -> testPrimStrings:bit_string(Rule), @@ -368,10 +367,10 @@ testPrimExternal(Config, Rule, Opts) -> asn1_test_lib:compile_all(["External", "PrimExternal"], Config, [Rule|Opts]), testPrimExternal:external(Rule), - ?only_ber(asn1_test_lib:compile_all(["PrimStrings", "BitStr"], Config, - [Rule|Opts])), - ?only_ber(testPrimStrings_cases(Rule)), - ?only_ber(testPrimStrings:more_strings(Rule)). + asn1_test_lib:compile_all(["PrimStrings", "BitStr"], Config, + [Rule|Opts]), + testPrimStrings_cases(Rule), + testPrimStrings:more_strings(Rule). testChoPrim(Config) -> test(Config, fun testChoPrim/3). testChoPrim(Config, Rule, Opts) -> @@ -634,9 +633,10 @@ c_syntax(Config) -> "SeqBadComma"]]. c_string(Config) -> - test(Config, fun c_string/3, [per, ber]). + test(Config, fun c_string/3). c_string(Config, Rule, Opts) -> - asn1_test_lib:compile("String", Config, [Rule|Opts]). + asn1_test_lib:compile("String", Config, [Rule|Opts]), + asn1ct:test('String'). c_implicit_before_choice(Config) -> test(Config, fun c_implicit_before_choice/3, [ber]). @@ -688,6 +688,8 @@ ber_other(Config) -> ber_other(Config, Rule, Opts) -> [module_test(M, Config, Rule, Opts) || M <- ber_modules()]. +der(Config) -> + asn1_test_lib:compile_all(ber_modules(), Config, [der]). module_test(M, Config, Rule, Opts) -> asn1_test_lib:compile(M, Config, [Rule|Opts]), @@ -740,13 +742,9 @@ value_test(Config, Rule, Opts) -> 'ObjIdValues':'mobileDomainId'()). value_bad_enum_test(Config) -> - case ?MODULE of - asn1_SUITE -> - {error, _} = asn1ct:compile(?config(data_dir, Config) - ++ "BadEnumValue1", - [{outdir, ?config(case_dir, Config)}]); - _ -> {skip, "Runs in asn1_SUITE only"} - end. + {error, _} = asn1ct:compile(?config(data_dir, Config) ++ + "BadEnumValue1", + [{outdir, ?config(case_dir, Config)}]). constructed(Config) -> test(Config, fun constructed/3, [ber]). @@ -861,18 +859,13 @@ testInvokeMod(Config, Rule, Opts) -> {ok, _Result2} = 'PrimStrings':encode('Bs1', [1, 0, 1, 0]). testExport(Config) -> - case ?MODULE of - asn1_SUITE -> - {error, {asn1, _Reason}} = - asn1ct:compile(filename:join(?config(data_dir, Config), - "IllegalExport"), - [{outdir, ?config(case_dir, Config)}]); - _ -> - {skip, "Runs in asn1_SUITE only"} - end. + {error, {asn1, _Reason}} = + asn1ct:compile(filename:join(?config(data_dir, Config), + "IllegalExport"), + [{outdir, ?config(case_dir, Config)}]). testImport(Config) -> - test(Config, fun testImport/3, [ber]). + test(Config, fun testImport/3). testImport(Config, Rule, Opts) -> {error, _} = asn1ct:compile(filename:join(?config(data_dir, Config), "ImportsFrom"), @@ -910,18 +903,14 @@ testOpenTypeImplicitTag(Config, Rule, Opts) -> testOpenTypeImplicitTag:main(Rule). duplicate_tags(Config) -> - case ?MODULE of - asn1_SUITE -> - DataDir = ?config(data_dir, Config), - CaseDir = ?config(case_dir, Config), - {error, {asn1, [{error, {type, _, _, 'SeqOpt1Imp', {asn1, {duplicates_of_the_tags, _}}}}]}} = - asn1ct:compile(filename:join(DataDir, "SeqOptional2"), - [abs, {outdir, CaseDir}]); - _ -> - {skip, "Runs in asn1_SUITE only"} - end. + DataDir = ?config(data_dir, Config), + CaseDir = ?config(case_dir, Config), + {error, {asn1, [{error, {type, _, _, 'SeqOpt1Imp', + {asn1, {duplicates_of_the_tags, _}}}}]}} = + asn1ct:compile(filename:join(DataDir, "SeqOptional2"), + [abs, {outdir, CaseDir}]). -rtUI(Config) -> test(Config, fun rtUI/3, [per,ber]). +rtUI(Config) -> test(Config, fun rtUI/3). rtUI(Config, Rule, Opts) -> asn1_test_lib:compile("Prim", Config, [Rule|Opts]), {ok, _} = asn1rt:info('Prim'). @@ -937,7 +926,7 @@ testINSTANCE_OF(Config, Rule, Opts) -> testINSTANCE_OF:main(Rule). testTCAP(Config) -> - test(Config, fun testTCAP/3, [ber]). + test(Config, fun testTCAP/3). testTCAP(Config, Rule, Opts) -> testTCAP:compile(Config, [Rule|Opts]), testTCAP:test(Rule, Config), @@ -988,11 +977,16 @@ test_driver_load(Config, Rule, Opts) -> test_ParamTypeInfObj(Config) -> asn1_test_lib:compile("IN-CS-1-Datatypes", Config, [ber]). -test_WS_ParamClass(Config) -> - asn1_test_lib:compile("InformationFramework", Config, [ber]). +test_WS_ParamClass(Config) -> test(Config, fun test_WS_ParamClass/3). +test_WS_ParamClass(Config, Rule, Opts) -> + asn1_test_lib:compile("InformationFramework", Config, [Rule|Opts]), + ?only_ber(testWSParamClass:main(Rule)), + ok. test_Defed_ObjectIdentifier(Config) -> - asn1_test_lib:compile("UsefulDefinitions", Config, [ber]). + test(Config, fun test_Defed_ObjectIdentifier/3). +test_Defed_ObjectIdentifier(Config, Rule, Opts) -> + asn1_test_lib:compile("UsefulDefinitions", Config, [Rule|Opts]). testSelectionType(Config) -> test(Config, fun testSelectionType/3). testSelectionType(Config, Rule, Opts) -> @@ -1020,7 +1014,7 @@ test_undecoded_rest(Config, Rule, Opts) -> test_undecoded_rest:test(undec_rest, Config). testTcapsystem(Config) -> - test(Config, fun testTcapsystem/3, [ber]). + test(Config, fun testTcapsystem/3). testTcapsystem(Config, Rule, Opts) -> testTcapsystem:compile(Config, [Rule|Opts]). @@ -1123,6 +1117,7 @@ test_modules() -> "Int", "MAP-commonDataTypes", "Null", + "NullTest", "Octetstr", "One", "P-Record", @@ -1264,189 +1259,6 @@ smp(Config) -> {skipped,"No smp support"} end. -per_performance(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - NifDir = filename:join(PrivDir,"nif"), - ErlDir = filename:join(PrivDir,"erl"), - file:make_dir(NifDir),file:make_dir(ErlDir), - - Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()}, - ok = testNBAPsystem:compile([{priv_dir,NifDir}|Config], [per]), - ok = testNBAPsystem:compile([{priv_dir,ErlDir}|Config], [per]), - - Modules = ['NBAP-CommonDataTypes', - 'NBAP-Constants', - 'NBAP-Containers', - 'NBAP-IEs', - 'NBAP-PDU-Contents', - 'NBAP-PDU-Discriptions'], - - - PreNif = fun() -> - code:add_patha(NifDir), - lists:foreach(fun(M) -> - code:purge(M), - code:load_file(M) - end,Modules) - end, - - PreErl = fun() -> - code:add_patha(ErlDir), - lists:foreach(fun(M) -> - code:purge(M), - code:load_file(M) - end,Modules) - end, - - Func = fun() -> - element(1,timer:tc( - asn1_wrapper,encode,['NBAP-PDU-Discriptions', - 'NBAP-PDU', - Msg])) - end, - - nif_vs_erlang_performance({{{PreNif,Func},{PreErl,Func}},100000,32}). - -ber_performance(Config) -> - - Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()}, - ok = testNBAPsystem:compile(Config, [ber]), - - - BerFun = fun() -> - {ok,B} = asn1_wrapper:encode('NBAP-PDU-Discriptions', - 'NBAP-PDU', Msg), - asn1_wrapper:decode( - 'NBAP-PDU-Discriptions', - 'NBAP-PDU', - B) - end, - nif_vs_erlang_performance({BerFun,100000,32}). - -cert_pem_performance(Config) when is_list(Config) -> - cert_pem_performance({100000, 32}); -cert_pem_performance({N,S}) -> - nif_vs_erlang_performance({fun pem_performance:cert_pem/0,N,S}). - -dsa_pem_performance(Config) when is_list(Config) -> - dsa_pem_performance({100000, 32}); -dsa_pem_performance({N,S}) -> - nif_vs_erlang_performance({fun pem_performance:dsa_pem/0,N,S}). - - -nif_vs_erlang_performance({{TC1,TC2},N,Sched}) -> - random:seed({123,456,789}), - io:format("Running a ~p sample with ~p max procs...~n~n",[N,Sched]), - - {True,False} = exec(TC1,TC2,Sched,N+1), - - io:format("~ndone!~n"), - - io:format("~n"),TStats = print_stats(strip(True,N div 20)), - io:format("~n"),FStats = print_stats(strip(False,N div 20)), - Str = io_lib:format("~nNifs are ~.3f% faster than erlang!~n", - [(element(2,FStats) - element(2,TStats)) / - element(2,FStats) * 100]), - io:format(Str), - {comment, lists:flatten(Str)}; -nif_vs_erlang_performance({T,N,Sched}) -> - PTC1 = fun() -> - application:set_env(asn1, nif_loadable, true) - end, - PTC2 = fun() -> - application:set_env(asn1, nif_loadable, false) - end, - TC = fun() -> - element(1,timer:tc(T)) - end, - nif_vs_erlang_performance({{{PTC1,TC},{PTC2,TC}},N,Sched}). - - -print_stats(Data) -> - Length = length(Data), - Mean = lists:sum(Data) / Length, - Variance = lists:foldl(fun(N,Acc) -> math:pow(N - Mean, 2)+Acc end, 0, Data), - StdDev = math:sqrt(Variance / Length), - Median = lists:nth(round(Length/2),Data), - Min = lists:min(Data), - Max = lists:max(Data), - if Length < 20 -> - io:format("Data: ~w~n",[Data]); - true -> - ok - end, - io:format("Length: ~p~nMean: ~p~nStdDev: ~p~nMedian: ~p~nMin: ~p~nMax: ~p~n", - [Length,Mean,StdDev,Median,Min,Max]), - {Length,Mean,StdDev,Median,Min,Max}. - -collect(Acc) -> - receive - {Tag,Val} -> - Prev = proplists:get_value(Tag,Acc,[]), - collect(lists:keystore(Tag,1,Acc,{Tag,[Val|Prev]})) - after 100 -> - Acc - end. - -exec(One,Two,Max,N) -> - exec(One,Two,Max,N,{[],[]}). -exec(_,_,_,1,{D1,D2}) -> - {lists:flatten(D1),lists:flatten(D2)}; -exec({PreOne,One} = O,{PreTwo,Two} = T,MaxProcs, N, {D1,D2}) -> - Num = random:uniform(round(N/2)), - if Num rem 3 == 0 -> - timer:sleep(Num rem 1000); - true -> - ok - end, - Procs = random:uniform(MaxProcs), - io:format("\tBatch: ~p items in ~p processes, ~p left~n",[Num,Procs,N-Num]), - if Num rem 2 == 1 -> - erlang:garbage_collect(), - PreOne(), - MoreOne = pexec(One, Num, Procs, []), - erlang:garbage_collect(), - PreTwo(), - MoreTwo = pexec(Two, Num, Procs, []); - true -> - erlang:garbage_collect(), - PreTwo(), - MoreTwo = pexec(Two, Num, Procs, []), - erlang:garbage_collect(), - PreOne(), - MoreOne = pexec(One, Num, Procs, []) - end, - exec(O,T,MaxProcs,N-Num,{[MoreOne|D1], - [MoreTwo|D2]}). - -pexec(_Fun, _, 0, []) -> - []; -pexec(Fun, _, 0, [{Ref,Pid}|Rest]) -> - receive - {data,D} -> - [D|pexec(Fun,0,0,[{Ref,Pid}|Rest])]; - {'DOWN', Ref, process, Pid, normal} -> - pexec(Fun, 0,0,Rest) - end; -pexec(Fun, 0, 1, AccProcs) -> - pexec(Fun, 0, 0, AccProcs); -pexec(Fun, N, 1, AccProcs) -> - [Fun()|pexec(Fun, N - 1, 1, AccProcs)]; -pexec(Fun, N, Procs, AccProcs) -> - S = self(), - Pid = spawn(fun() -> - S ! {data,pexec(Fun,N,1,[])} - end), - Ref = erlang:monitor(process, Pid), - pexec(Fun, N, Procs - 1, [{Ref,Pid}|AccProcs]). - -strip(Data,Num) -> - {_,R} = lists:split(Num,lists:sort(Data)), - element(2,lists:split(Num,lists:reverse(R))). - -faster(A,B) -> - (B - A)/B * 100. - enc_dec(1, Msg, N) -> worker_loop(N, Msg); enc_dec(NumOfProcs,Msg, N) -> diff --git a/lib/asn1/test/asn1_SUITE_data/Def.py b/lib/asn1/test/asn1_SUITE_data/Def.py deleted file mode 100644 index ff08ed6386..0000000000 --- a/lib/asn1/test/asn1_SUITE_data/Def.py +++ /dev/null @@ -1,31 +0,0 @@ -Def DEFINITIONS IMPLICIT TAGS ::= - -BEGIN - -Def1 ::= SEQUENCE -{ - bool0 [0] BOOLEAN, - bool1 [1] BOOLEAN DEFAULT false, - bool2 [2] BOOLEAN DEFAULT false, - bool3 [3] BOOLEAN DEFAULT false -} - - -Def2 ::= SEQUENCE -{ - bool10 [10] BOOLEAN, - bool11 [11] BOOLEAN DEFAULT false, - bool12 [12] BOOLEAN DEFAULT false, - bool13 [13] BOOLEAN -} - - -Def3 ::= SEQUENCE -{ - bool30 [30] BOOLEAN DEFAULT false, - bool31 [31] BOOLEAN DEFAULT false, - bool32 [32] BOOLEAN DEFAULT false, - bool33 [33] BOOLEAN DEFAULT false -} - -END diff --git a/lib/asn1/test/asn1_SUITE_data/NullTest.asn1 b/lib/asn1/test/asn1_SUITE_data/NullTest.asn1 new file mode 100644 index 0000000000..041b20a4c1 --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/NullTest.asn1 @@ -0,0 +1,14 @@ +NullTest DEFINITIONS ::= +BEGIN + +NullTestData ::= SEQUENCE { + body NullBody, + tail INTEGER +} + +NullBody ::= CHOICE { + null [0] NULL, + notNull [1] INTEGER +} + +END diff --git a/lib/asn1/test/asn1_SUITE_data/Opt.py b/lib/asn1/test/asn1_SUITE_data/Opt.py deleted file mode 100644 index 48c2a09b64..0000000000 --- a/lib/asn1/test/asn1_SUITE_data/Opt.py +++ /dev/null @@ -1,31 +0,0 @@ -Opt DEFINITIONS IMPLICIT TAGS ::= - -BEGIN - -Opt1 ::= SEQUENCE -{ - bool0 [0] BOOLEAN, - bool1 [1] BOOLEAN OPTIONAL, - bool2 [2] BOOLEAN OPTIONAL, - bool3 [3] BOOLEAN OPTIONAL -} - - -Opt2 ::= SEQUENCE -{ - bool10 [10] BOOLEAN, - bool11 [11] BOOLEAN OPTIONAL, - bool12 [12] BOOLEAN OPTIONAL, - bool13 [13] BOOLEAN -} - - -Opt3 ::= SEQUENCE -{ - bool30 [30] BOOLEAN OPTIONAL, - bool31 [31] BOOLEAN OPTIONAL, - bool32 [32] BOOLEAN OPTIONAL, - bool33 [33] BOOLEAN OPTIONAL -} - -END diff --git a/lib/asn1/test/asn1_SUITE_data/SeqOf.py b/lib/asn1/test/asn1_SUITE_data/SeqOf.py deleted file mode 100644 index c941418934..0000000000 --- a/lib/asn1/test/asn1_SUITE_data/SeqOf.py +++ /dev/null @@ -1,45 +0,0 @@ -SeqOf DEFINITIONS IMPLICIT TAGS ::= - -BEGIN - - -Seq1 ::= SEQUENCE -{ - bool1 BOOLEAN, - int1 INTEGER, - seq1 SEQUENCE OF SeqIn DEFAULT {} -} - -Seq2 ::= SEQUENCE -{ - seq2 SEQUENCE OF SeqIn DEFAULT {}, - bool2 BOOLEAN, - int2 INTEGER -} - -Seq3 ::= SEQUENCE -{ - bool3 BOOLEAN, - seq3 SEQUENCE OF SeqIn DEFAULT {}, - int3 INTEGER -} - -Seq4 ::= SEQUENCE -{ - seq41 [41] SEQUENCE OF SeqIn DEFAULT {}, - seq42 [42] SEQUENCE OF SeqIn DEFAULT {}, - seq43 [43] SEQUENCE OF SeqIn DEFAULT {} -} - - - -SeqIn ::= SEQUENCE -{ - boolIn BOOLEAN, - intIn INTEGER -} - - - - -END diff --git a/lib/asn1/test/asn1_SUITE_data/SetOf.py b/lib/asn1/test/asn1_SUITE_data/SetOf.py deleted file mode 100644 index 4e2ea16fcc..0000000000 --- a/lib/asn1/test/asn1_SUITE_data/SetOf.py +++ /dev/null @@ -1,42 +0,0 @@ -SetOf DEFINITIONS IMPLICIT TAGS ::= - -BEGIN - - -Set1 ::= SET -{ - bool1 BOOLEAN, - int1 INTEGER, - set1 SET OF SetIn DEFAULT {} -} - -Set2 ::= SET -{ - set2 SET OF SetIn DEFAULT {}, - bool2 BOOLEAN, - int2 INTEGER -} - -Set3 ::= SET -{ - bool3 BOOLEAN, - set3 SET OF SetIn DEFAULT {}, - int3 INTEGER -} - -Set4 ::= SET -{ - set41 [41] SET OF SetIn DEFAULT {}, - set42 [42] SET OF SetIn DEFAULT {}, - set43 [43] SET OF SetIn DEFAULT {} -} - - - -SetIn ::= SET -{ - boolIn BOOLEAN, - intIn INTEGER -} - -END diff --git a/lib/asn1/test/testTCAP.erl b/lib/asn1/test/testTCAP.erl index b723995e40..aba13c94de 100644 --- a/lib/asn1/test/testTCAP.erl +++ b/lib/asn1/test/testTCAP.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2012. All Rights Reserved. +%% Copyright Ericsson AB 2003-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -37,7 +37,7 @@ compile_asn1config(Config, Options) -> asn1_test_lib:compile_all(Files, Config, Options), asn1_test_lib:compile_erlang("TCAPPackage_msg", Config, []). -test(ber=Erule,_Config) -> +test(Erule,_Config) -> % ?line OutDir = ?config(priv_dir,Config), %% testing OTP-4798, open type encoded with indefinite length ?line {ok,_Res} = asn1_wrapper:decode('TCAPMessages-simple','MessageType', val_OTP_4798(Erule)), diff --git a/lib/asn1/test/testWSParamClass.erl b/lib/asn1/test/testWSParamClass.erl new file mode 100644 index 0000000000..ae67ca8b81 --- /dev/null +++ b/lib/asn1/test/testWSParamClass.erl @@ -0,0 +1,17 @@ +-module(testWSParamClass). +-export([main/1]). + +main(_) -> + IF = 'InformationFramework', + roundtrip({'Attribute',IF:'id-at-objectClass'(), + [IF:'id-at-objectClass'()], + asn1_NOVALUE}), + roundtrip({'Attribute',IF:'id-at-objectClass'(), + [],[]}), + ok. + +roundtrip(Data) -> + IF = 'InformationFramework', + {ok,Enc} = asn1_wrapper:encode(IF, 'Attribute', Data), + {ok,Data} = IF:decode('Attribute', Enc), + ok. diff --git a/lib/asn1/test/testX420.erl b/lib/asn1/test/testX420.erl index 045660a8e2..70bdb0640d 100644 --- a/lib/asn1/test/testX420.erl +++ b/lib/asn1/test/testX420.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/asn1/test/test_modified_x420.erl b/lib/asn1/test/test_modified_x420.erl index ae9d1989fb..a525fd6ae1 100644 --- a/lib/asn1/test/test_modified_x420.erl +++ b/lib/asn1/test/test_modified_x420.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/asn1/vsn.mk b/lib/asn1/vsn.mk index 8f4f68c855..aaa060c806 100644 --- a/lib/asn1/vsn.mk +++ b/lib/asn1/vsn.mk @@ -1,2 +1,2 @@ #next version number to use is 2.0 -ASN1_VSN = 2.0 +ASN1_VSN = 2.0.1 diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml index 4fa92d5583..736486350b 100644 --- a/lib/common_test/doc/src/cover_chapter.xml +++ b/lib/common_test/doc/src/cover_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2012</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -74,12 +74,6 @@ executed during the test. In overview mode, only the code coverage overview page gets printed.</p> - <p><em>Note:</em> Currently, for Common Test to be able to print - code coverage HTML files for the modules included in the - analysis, the source code files of these modules must be - located in the same directory as the corresponding <c>.beam</c> - files. This is a limitation that will be removed later.</p> - <p>You can choose to export and import code coverage data between tests. If you specify the name of an export file in the cover specification file, Common Test will export collected coverage diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index 198290c1be..d871908952 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>2007</year><year>2012</year> + <year>2007</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 8cbcbad8b2..5d2c065385 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -32,6 +32,69 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.7.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + If an event handler installed in the CT Master event + manager took too long to respond during the termination + phase, CT Master crashed because of a timeout after 5 + secs. This would leave the system in a bad state. The + problem has been solved by means of a 30 min timeout + value and if CT Master gets a timeout after that time, it + now kills the event manager and shuts down properly.</p> + <p> + Own Id: OTP-10634 Aux Id: kunagi-347 [258] </p> + </item> + <item> + <p> + Printing with any of the ct printout functions from an + event handler installed by Common Test, would cause a + deadlock. This problem has been solved.</p> + <p> + Own Id: OTP-10826 Aux Id: seq12250 </p> + </item> + <item> + <p> + Using the force_stop flag/option to interrupt a test run + caused a crash in Common Test. This problem has been + solved.</p> + <p> + Own Id: OTP-10832</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Removed depricated run_test program, use ct_run instead.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-9052</p> + </item> + </list> + </section> + + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p> + Test case execution time increases with size of test run.</p> + <p> + Own Id: OTP-10855</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.7</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 5924930072..0b204a681a 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -397,9 +397,9 @@ tc_print(Category,Format,Args) -> %%% <p>This function is called by <code>ct</code> when printing %%% stuff from a testcase on the user console.</p> tc_print(Category,Importance,Format,Args) -> - VLvl = case ct_util:get_testdata({verbosity,Category}) of + VLvl = case ct_util:get_verbosity(Category) of undefined -> - ct_util:get_testdata({verbosity,'$unspecified'}); + ct_util:get_verbosity('$unspecified'); {error,bad_invocation} -> ?MAX_VERBOSITY; Val -> @@ -1475,8 +1475,9 @@ count_cases(Dir) -> write_summary(SumFile, Summary), Summary end; - {error, _Reason} -> - io:format("\nFailed to read ~p (skipped)\n", [LogFile]), + {error, Reason} -> + io:format("\nFailed to read ~p: ~p (skipped)\n", + [LogFile,Reason]), error end end. diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index e516f635d2..b42ff73846 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -428,7 +428,7 @@ master_loop(#state{node_ctrl_pids=[], log(all,"TEST RESULTS",Str,[]), log(all,"Info","Updating log files",[]), refresh_logs(LogDirs,[]), - + ct_master_event:stop(), ct_master_logs:stop(), ok; diff --git a/lib/common_test/src/ct_master_event.erl b/lib/common_test/src/ct_master_event.erl index 5877b7c6f2..fd97ab16f7 100644 --- a/lib/common_test/src/ct_master_event.erl +++ b/lib/common_test/src/ct_master_event.erl @@ -66,16 +66,30 @@ add_handler(Args) -> %% Description: Stops the event manager %%-------------------------------------------------------------------- stop() -> - flush(), - gen_event:stop(?CT_MEVMGR_REF). + case flush() of + {error,Reason} -> + ct_master_logs:log("Error", + "No response from CT Master Event.\n" + "Reason = ~p\n" + "Terminating now!\n",[Reason]), + %% communication with event manager fails, kill it + catch exit(whereis(?CT_MEVMGR_REF), kill); + _ -> + gen_event:stop(?CT_MEVMGR_REF) + end. flush() -> - case gen_event:call(?CT_MEVMGR_REF,?MODULE,flush) of + try gen_event:call(?CT_MEVMGR_REF,?MODULE,flush,1800000) of flushing -> timer:sleep(1), flush(); done -> - ok + ok; + Error = {error,_} -> + Error + catch + _:Reason -> + {error,Reason} end. %%-------------------------------------------------------------------- diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index c0bdbb2a09..49f00429ae 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -2291,8 +2291,12 @@ add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) -> {'EXIT',_} -> CleanUp; _ -> - wait_for_idle(), - add_jobs(Tests, Skip, Opts, CleanUp) + case wait_for_idle() of + ok -> + add_jobs(Tests, Skip, Opts, CleanUp); + _ -> + CleanUp + end end; add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opts, CleanUp) when is_atom(Suite) -> @@ -2305,8 +2309,12 @@ add_jobs([{TestDir,Suites,all}|Tests], Skip, {'EXIT',_} -> CleanUp; _ -> - wait_for_idle(), - add_jobs(Tests, Skip, Opts, CleanUp) + case wait_for_idle() of + ok -> + add_jobs(Tests, Skip, Opts, CleanUp); + _ -> + CleanUp + end end; add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) -> case maybe_interpret(Suite, all, Opts) of @@ -2318,8 +2326,12 @@ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) -> {'EXIT',_} -> CleanUp; _ -> - wait_for_idle(), - add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) + case wait_for_idle() of + ok -> + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); + _ -> + CleanUp + end end; Error -> Error @@ -2358,8 +2370,12 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when {'EXIT',_} -> CleanUp; _ -> - wait_for_idle(), - add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) + case wait_for_idle() of + ok -> + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); + _ -> + CleanUp + end end; Error -> Error @@ -2384,8 +2400,12 @@ add_jobs([{TestDir,Suite,Cases}|Tests], {'EXIT',_} -> CleanUp; _ -> - wait_for_idle(), - add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) + case wait_for_idle() of + ok -> + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); + _ -> + CleanUp + end end; Error -> Error @@ -2401,8 +2421,12 @@ add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) - {'EXIT',_} -> CleanUp; _ -> - wait_for_idle(), - add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) + case wait_for_idle() of + ok -> + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); + _ -> + CleanUp + end end; Error -> Error @@ -2412,7 +2436,13 @@ add_jobs([], _, _, CleanUp) -> wait_for_idle() -> ct_util:update_last_run_index(), - Notify = fun(Me) -> Me ! idle end, + Notify = fun(Me,IdleState) -> Me ! {idle,IdleState}, + receive + {Me,proceed} -> ok + after + 30000 -> ok + end + end, case catch test_server_ctrl:idle_notify(Notify) of {'EXIT',_} -> error; @@ -2420,11 +2450,14 @@ wait_for_idle() -> %% so we don't hang forever if test_server dies Ref = erlang:monitor(process, TSPid), Result = receive - idle -> ok; + {idle,abort} -> aborted; + {idle,_} -> ok; {'DOWN', Ref, _, _, _} -> error end, erlang:demonitor(Ref, [flush]), ct_util:update_last_run_index(), + %% let test_server_ctrl proceed (and possibly shut down) now + TSPid ! {self(),proceed}, Result end. @@ -2921,11 +2954,11 @@ opts2args(EnvStartOpts) -> [{event_handler_init,[atom_to_list(EH),ArgStr]}]; ({event_handler,{EHs,Arg}}) when is_list(EHs) -> ArgStr = lists:flatten(io_lib:format("~p", [Arg])), - Strs = lists:map(fun(EH) -> - [atom_to_list(EH), - ArgStr,"and"] - end, EHs), - [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), + Strs = lists:flatmap(fun(EH) -> + [atom_to_list(EH), + ArgStr,"and"] + end, EHs), + [_LastAnd | StrsR] = lists:reverse(Strs), [{event_handler_init,lists:reverse(StrsR)}]; ({logopts,LOs}) when is_list(LOs) -> [{logopts,[atom_to_list(LO) || LO <- LOs]}]; diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index e341391a91..71b03c0ea6 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -1020,17 +1020,6 @@ add_tests([],Spec) -> % done %% check if it's a CT term that has bad format or if the user seems to %% have added something of his/her own, which we'll let pass if relaxed %% mode is enabled. -check_term(Atom) when is_atom(Atom) -> - Valid = valid_terms(), - case lists:member(Atom,Valid) of - true -> - valid; - false -> % ignore - case get(relaxed) of - true -> invalid; - false -> throw({error,{undefined_term_in_spec,Atom}}) - end - end; check_term(Term) when is_tuple(Term) -> Size = size(Term), [Name|_] = tuple_to_list(Term), @@ -1059,9 +1048,7 @@ check_term(Term) when is_tuple(Term) -> throw({error,{undefined_term_in_spec,Term}}) end end - end; -check_term(Other) -> - throw({error,{undefined_term_in_spec,Other}}). + end. %% specific data handling before saving in testspec record, e.g. %% converting relative paths to absolute for directories and files diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index 0f2b2081d9..2e7e731595 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -39,7 +39,8 @@ delete_suite_data/0, delete_suite_data/1, match_delete_suite_data/1, delete_testdata/0, delete_testdata/1, set_testdata/1, get_testdata/1, get_testdata/2, - set_testdata_async/1, update_testdata/2, update_testdata/3]). + set_testdata_async/1, update_testdata/2, update_testdata/3, + set_verbosity/1, get_verbosity/1]). -export([override_silence_all_connections/0, override_silence_connections/1, get_overridden_silenced_connections/0, @@ -128,6 +129,10 @@ do_start(Parent, Mode, LogDir, Verbosity) -> create_table(?conn_table,#conn.handle), create_table(?board_table,2), create_table(?suite_table,#suite_data.key), + + create_table(?verbosity_table,1), + [ets:insert(?verbosity_table,{Cat,Lvl}) || {Cat,Lvl} <- Verbosity], + {ok,StartDir} = file:get_cwd(), case file:set_cwd(LogDir) of ok -> ok; @@ -202,7 +207,7 @@ do_start(Parent, Mode, LogDir, Verbosity) -> self() ! {{stop,{self(),{user_error,CTHReason}}}, {Parent,make_ref()}} end, - loop(Mode, [{{verbosity,Cat},Lvl} || {Cat,Lvl} <- Verbosity], StartDir). + loop(Mode, [], StartDir). create_table(TableName,KeyPos) -> create_table(TableName,set,KeyPos). @@ -278,6 +283,19 @@ reset_cwd() -> get_start_dir() -> call(get_start_dir). +%% handle verbosity outside ct_util_server (let the client read +%% the verbosity table) to avoid possible deadlock situations +set_verbosity(Elem = {_Category,_Level}) -> + ets:insert(?verbosity_table, Elem), + ok. +get_verbosity(Category) -> + case ets:lookup(?verbosity_table, Category) of + [{Category,Level}] -> + Level; + _ -> + undefined + end. + loop(Mode,TestData,StartDir) -> receive {update_last_run_index,From} -> @@ -377,6 +395,7 @@ loop(Mode,TestData,StartDir) -> ets:delete(?conn_table), ets:delete(?board_table), ets:delete(?suite_table), + ets:delete(?verbosity_table), ct_logs:close(Info, StartDir), ct_event:stop(), ct_config:stop(), diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index c9c6514fa4..7c01e17c36 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2012. All Rights Reserved. +%% Copyright Ericsson AB 2003-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,7 @@ -define(conn_table,ct_connections). -define(board_table,ct_boards). -define(suite_table,ct_suite_data). +-define(verbosity_table,ct_verbosity_table). -record(conn, {handle, targetref, diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 760cc20410..bd746f87a7 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2012. All Rights Reserved. +# Copyright Ericsson AB 2008-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 86ec71c98e..3881ced17d 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl index fb47f2bcb9..bcbf972a36 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl index 40a54b9f99..446dd8bfdf 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index 0f336d2d79..7408cbe376 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 7c33fd404d..5e109e98e9 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -312,8 +312,10 @@ wait_for_ct_stop(Retries, CTNode) -> undefined -> true; Pid -> + Info = (catch process_info(Pid)), test_server:format(0, "Waiting for CT (~p) to finish (~p)...", [Pid,Retries]), + test_server:format(0, "Process info for ~p:~n~p", [Info]), timer:sleep(5000), wait_for_ct_stop(Retries-1, CTNode) end. @@ -328,12 +330,17 @@ handle_event(EH, Event) -> start_event_receiver(Config) -> CTNode = proplists:get_value(ct_node, Config), - spawn_link(CTNode, fun() -> er() end). + Level = proplists:get_value(trace_level, Config), + ER = spawn_link(CTNode, fun() -> er() end), + test_server:format(Level, "~nEvent receiver ~w started!~n", [ER]), + ER. get_events(_, Config) -> CTNode = proplists:get_value(ct_node, Config), + Level = proplists:get_value(trace_level, Config), {event_receiver,CTNode} ! {self(),get_events}, Events = receive {event_receiver,Evs} -> Evs end, + test_server:format(Level, "Stopping event receiver!~n", []), {event_receiver,CTNode} ! stop, Events. diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl index beaba0eaac..518352e87c 100644 --- a/lib/common_test/test/ct_testspec_2_SUITE.erl +++ b/lib/common_test/test/ct_testspec_2_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl index e2f57c2c64..6b4b729552 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl index bbe79ed3fe..b8216c3596 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl index 810298d348..7c51aca246 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl index 9348cd8caf..36c1b4279b 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl index a92018ec70..3f6336c7e2 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl index d01fac3144..d836ab57c1 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/test/ct_verbosity_SUITE.erl b/lib/common_test/test/ct_verbosity_SUITE.erl index 349319de94..32488b1db9 100644 --- a/lib/common_test/test/ct_verbosity_SUITE.erl +++ b/lib/common_test/test/ct_verbosity_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -44,8 +44,11 @@ %% there will be clashes with logging processes etc). %%-------------------------------------------------------------------- init_per_suite(Config) -> - Config1 = ct_test_support:init_per_suite(Config), - Config1. + DataDir = ?config(data_dir, Config), + EvH = filename:join(DataDir,"simple_evh.erl"), + ct:pal("Compiling ~s: ~p", [EvH,compile:file(EvH,[{outdir,DataDir}, + debug_info])]), + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). end_per_suite(Config) -> ct_test_support:end_per_suite(Config). @@ -56,7 +59,8 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> [{timetrap,{minutes,2}}, + {ct_hooks,[ts_install_cth]}]. all() -> [ @@ -67,7 +71,8 @@ all() -> change_default, combine_categories, testspec_only, - merge_with_testspec + merge_with_testspec, + possible_deadlock ]. %%-------------------------------------------------------------------- @@ -173,6 +178,17 @@ merge_with_testspec(Config) -> ok = execute(TC, Opts, ERPid, Config). %%%----------------------------------------------------------------- +%%% +possible_deadlock(Config) -> + TC = possible_deadlock, + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "io_test_SUITE"), + {Opts,ERPid} = setup([{suite,Suite},{label,TC}, + {event_handler,[simple_evh]}], Config), + ok = execute(TC, Opts, ERPid, Config). + + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -180,7 +196,14 @@ setup(Test, Config) -> Opts0 = ct_test_support:get_opts(Config), Level = ?config(trace_level, Config), EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], - Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + Opts = + case proplists:get_value(event_handler, Test) of + undefined -> + Opts0 ++ [{event_handler,{?eh,EvHArgs}} | Test]; + EvHs -> + Opts0 ++ [{event_handler,{[?eh|EvHs],EvHArgs}} | + proplists:delete(event_handler, Test)] + end, ERPid = ct_test_support:start_event_receiver(Config), {Opts,ERPid}. diff --git a/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl b/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl new file mode 100644 index 0000000000..3e744f2596 --- /dev/null +++ b/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl @@ -0,0 +1,171 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Framework Event Handler +%%% +%%% <p>This module implements an event handler that CT uses to +%%% handle status and progress notifications during test runs. +%%% The notifications are handled locally (per node) and passed +%%% on to ct_master when CT runs in distributed mode. This +%%% module may be used as a template for other event handlers +%%% that can be plugged in to handle local logging and reporting.</p> +-module(simple_evh). + +-behaviour(gen_event). + +%% gen_event callbacks +-export([init/1, handle_event/2, handle_call/2, + handle_info/2, terminate/2, code_change/3]). + +-include_lib("common_test/include/ct_event.hrl"). +-include_lib("common_test/src/ct_util.hrl"). + +%%==================================================================== +%% gen_event callbacks +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} +%% Description: Whenever a new event handler is added to an event manager, +%% this function is called to initialize the event handler. +%%-------------------------------------------------------------------- +init(_) -> + io:format("Event handler ~w started!~n", [?MODULE]), + {ok,[]}. + +%%-------------------------------------------------------------------- +%% Function: +%% handle_event(Event, State) -> {ok, State} | +%% {swap_handler, Args1, State1, Mod2, Args2} | +%% remove_handler +%% Description:Whenever an event manager receives an event sent using +%% gen_event:notify/2 or gen_event:sync_notify/2, this function is called for +%% each installed event handler to handle the event. +%%-------------------------------------------------------------------- +handle_event(Event = #event{name = test_stats},State) -> + %% this could cause a deadlock + ct:pal("~p: ~p~n", [Event#event.name,Event#event.data]), + {ok,State}; +handle_event(_Event,State) -> + {ok,State}. + +%%============================== EVENTS ============================== +%% +%% Name = test_start +%% Data = {StartTime,LogDir} +%% +%% Name = start_info +%% Data = {Tests,Suites,Cases} +%% Tests = Suites = Cases = integer() +%% +%% Name = test_done +%% Data = EndTime +%% +%% Name = start_make +%% Data = Dir +%% +%% Name = finished_make +%% Data = Dir +%% +%% Name = tc_start +%% Data = {Suite,CaseOrGroup} +%% CaseOrGroup = atom() | {Conf,GroupName,GroupProperties} +%% Conf = init_per_group | end_per_group +%% GroupName = atom() +%% GroupProperties = list() +%% +%% Name = tc_done +%% Data = {Suite,CaseOrGroup,Result} +%% CaseOrGroup = atom() | {Conf,GroupName,GroupProperties} +%% Conf = init_per_group | end_per_group +%% GroupName = atom() +%% GroupProperties = list() +%% Result = ok | {skipped,Reason} | {failed,Reason} +%% +%% Name = tc_user_skip +%% Data = {Suite,Case,Comment} +%% Comment = string() +%% +%% Name = tc_auto_skip +%% Data = {Suite,Case,Comment} +%% Comment = string() +%% +%% Name = test_stats +%% Data = {Ok,Failed,Skipped} +%% Ok = Failed = integer() +%% Skipped = {UserSkipped,AutoSkipped} +%% UserSkipped = AutoSkipped = integer() +%% +%% Name = start_logging +%% Data = CtRunDir +%% +%% Name = stop_logging +%% Data = [] +%% +%% Name = start_write_file +%% Data = FullNameFile +%% +%% Name = finished_write_file +%% Data = FullNameFile +%% +%% Name = +%% Data = +%% + +%%-------------------------------------------------------------------- +%% Function: +%% handle_call(Request, State) -> {ok, Reply, State} | +%% {swap_handler, Reply, Args1, State1, +%% Mod2, Args2} | +%% {remove_handler, Reply} +%% Description: Whenever an event manager receives a request sent using +%% gen_event:call/3,4, this function is called for the specified event +%% handler to handle the request. +%%-------------------------------------------------------------------- +handle_call(_Req, State) -> + Reply = ok, + {ok, Reply, State}. + +%%-------------------------------------------------------------------- +%% Function: +%% handle_info(Info, State) -> {ok, State} | +%% {swap_handler, Args1, State1, Mod2, Args2} | +%% remove_handler +%% Description: This function is called for each installed event handler when +%% an event manager receives any other message than an event or a synchronous +%% request (or a system message). +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description:Whenever an event handler is deleted from an event manager, +%% this function is called. It should be the opposite of Module:init/1 and +%% do any necessary cleaning up. +%%-------------------------------------------------------------------- +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index c92fb2ca37..87d762b697 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.7 +COMMON_TEST_VSN = 1.7.1 diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index a9579445bc..94fea84557 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,6 +31,53 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 4.9.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The compiler would crash attempting to compile + expressions such as "element(2, not_tuple)".</p> + <p> + Own Id: OTP-10794</p> + </item> + <item> + <p> + Forbid multiple values in Core Erlang sequence arguments. + Thanks to Jos� Valim and Anthony Ramine.</p> + <p> + Own Id: OTP-10818</p> + </item> + <item> + <p> + An unsafe optimization would cause the compiler to crash + with an internal error for certain complex code + sequences.</p> + <p> + Own Id: OTP-10825 Aux Id: seq12247 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>Integers in expression that will give a floating point + result (such as "<c>X / 2</c>" will now be converted to + floating point at compile-time. (Suggested by Richard + O'Keefe.)</p> + <p>Identical floating points constans in a module will + now be coalesced to one entry in the constant pool.</p> + <p> + Own Id: OTP-10788</p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 4.9</title> <section><title>Improvements and New Features</title> diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index 98967e651f..112b087f3c 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 67d756c45c..1a8bbcee22 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2012. All Rights Reserved. +%% Copyright Ericsson AB 2000-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 372923a5cf..3b51216a6c 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% Copyright Ericsson AB 1999-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl index cc07f0b9bc..3ad3c8c690 100644 --- a/lib/compiler/src/erl_bifs.erl +++ b/lib/compiler/src/erl_bifs.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2011. All Rights Reserved. +%% Copyright Ericsson AB 2001-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/compiler/test/compilation_SUITE.erl b/lib/compiler/test/compilation_SUITE.erl index a62ec7ce79..8e7b7292cc 100644 --- a/lib/compiler/test/compilation_SUITE.erl +++ b/lib/compiler/test/compilation_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index 8c60ce6489..90d2bff0ad 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 4.9 +COMPILER_VSN = 4.9.1 diff --git a/lib/cosEvent/doc/src/notes.xml b/lib/cosEvent/doc/src/notes.xml index e8877c2078..b0ac50ab10 100644 --- a/lib/cosEvent/doc/src/notes.xml +++ b/lib/cosEvent/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1999</year><year>2011</year> + <year>1999</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/cosEventDomain/doc/src/notes.xml b/lib/cosEventDomain/doc/src/notes.xml index db7f024b23..565b501148 100644 --- a/lib/cosEventDomain/doc/src/notes.xml +++ b/lib/cosEventDomain/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2001</year><year>2011</year> + <year>2001</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/cosFileTransfer/doc/src/notes.xml b/lib/cosFileTransfer/doc/src/notes.xml index 995aabfdee..56a85f2060 100644 --- a/lib/cosFileTransfer/doc/src/notes.xml +++ b/lib/cosFileTransfer/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2000</year><year>2011</year> + <year>2000</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/cosNotification/doc/src/notes.xml b/lib/cosNotification/doc/src/notes.xml index f1f70e566e..babf29d4a9 100644 --- a/lib/cosNotification/doc/src/notes.xml +++ b/lib/cosNotification/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2000</year><year>2011</year> + <year>2000</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/cosProperty/doc/src/notes.xml b/lib/cosProperty/doc/src/notes.xml index 22fab68cc3..eaf1100c4e 100644 --- a/lib/cosProperty/doc/src/notes.xml +++ b/lib/cosProperty/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2000</year><year>2011</year> + <year>2000</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/cosTime/doc/src/notes.xml b/lib/cosTime/doc/src/notes.xml index 2a65e000ec..f468f8aa53 100644 --- a/lib/cosTime/doc/src/notes.xml +++ b/lib/cosTime/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2000</year><year>2011</year> + <year>2000</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/cosTransactions/doc/src/notes.xml b/lib/cosTransactions/doc/src/notes.xml index d74e9ca3a6..e375338a8f 100644 --- a/lib/cosTransactions/doc/src/notes.xml +++ b/lib/cosTransactions/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1999</year><year>2011</year> + <year>1999</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 4936cae8ca..bd1f179f7d 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1999</year><year>2012</year> + <year>1999</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/debugger/doc/src/notes.xml b/lib/debugger/doc/src/notes.xml index d38d51075f..d5eef5d6a2 100644 --- a/lib/debugger/doc/src/notes.xml +++ b/lib/debugger/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -32,6 +32,25 @@ <p>This document describes the changes made to the Debugger application.</p> +<section><title>Debugger 3.2.10</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The +pc flag to erl can be used to set the range of + characters considered printable. This affects how the + shell and io:format("~tp",...) functionality does + heuristic string detection. More can be read in STDLIB + users guide.</p> + <p> + Own Id: OTP-10884</p> + </item> + </list> + </section> + +</section> + <section><title>Debugger 3.2.9</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/debugger/src/dbg_wx_trace.erl b/lib/debugger/src/dbg_wx_trace.erl index bd92cb4b42..eaea01822c 100644 --- a/lib/debugger/src/dbg_wx_trace.erl +++ b/lib/debugger/src/dbg_wx_trace.erl @@ -687,7 +687,7 @@ meta_cmd({trace_output, Str}, State) -> %% Reply on a user command meta_cmd({eval_rsp, Res}, State) -> - Str = io_lib:print(Res), + Str = io_lib_pretty:print(Res,[{encoding,unicode}]), dbg_wx_trace_win:eval_output(State#state.win, [$<,Str,10], normal), State. diff --git a/lib/debugger/src/dbg_wx_trace_win.erl b/lib/debugger/src/dbg_wx_trace_win.erl index 8b206ccd78..e54ce3913f 100644 --- a/lib/debugger/src/dbg_wx_trace_win.erl +++ b/lib/debugger/src/dbg_wx_trace_win.erl @@ -853,7 +853,7 @@ handle_event(#wx{id=?EVAL_ENTRY, event=#wxCommand{type=command_text_enter}}, handle_event(#wx{event=#wxList{type=command_list_item_selected, itemIndex=Row}},Wi) -> Bs = get(bindings), {Var,Val} = lists:nth(Row+1, Bs), - Str = io_lib:format("< ~s = ~p~n", [Var, Val]), + Str = io_lib:format("< ~s = ~lp~n", [Var, Val]), eval_output(Wi, Str, bold), ignore; handle_event(#wx{event=#wxList{type=command_list_item_activated, itemIndex=Row}},_Wi) -> diff --git a/lib/debugger/src/dbg_wx_win.erl b/lib/debugger/src/dbg_wx_win.erl index 3cb6edd953..eceacd7c88 100644 --- a/lib/debugger/src/dbg_wx_win.erl +++ b/lib/debugger/src/dbg_wx_win.erl @@ -254,7 +254,7 @@ notify(Win,Message) -> entry(Parent, Title, Prompt, {Type, Value}) -> Ted = wxTextEntryDialog:new(Parent, to_string(Prompt), [{caption, to_string(Title)}, - {value, to_string("~999999tp",Value)}]), + {value, to_string("~999999tp",[Value])}]), case wxDialog:showModal(Ted) of ?wxID_OK -> diff --git a/lib/debugger/vsn.mk b/lib/debugger/vsn.mk index b22b379cab..58d462f409 100644 --- a/lib/debugger/vsn.mk +++ b/lib/debugger/vsn.mk @@ -1 +1 @@ -DEBUGGER_VSN = 3.2.9 +DEBUGGER_VSN = 3.2.10 diff --git a/lib/dialyzer/doc/src/dialyzer.xml b/lib/dialyzer/doc/src/dialyzer.xml index 5e0c9b51e3..0ac96e8ac9 100644 --- a/lib/dialyzer/doc/src/dialyzer.xml +++ b/lib/dialyzer/doc/src/dialyzer.xml @@ -158,6 +158,9 @@ <item>Make Dialyzer a bit more quiet.</item> <tag><c><![CDATA[--verbose]]></c></tag> <item>Make Dialyzer a bit more verbose.</item> + <tag><c><![CDATA[--statistics]]></c></tag> + <item>Prints information about the progress of execution (analysis phases, + time spent in each and size of the relative input).</item> <tag><c><![CDATA[--build_plt]]></c></tag> <item>The analysis starts from an empty plt and creates a new one from the files specified with <c><![CDATA[-c]]></c> and @@ -228,6 +231,9 @@ match.</item> <tag><c><![CDATA[-Wno_opaque]]></c></tag> <item>Suppress warnings for violations of opaqueness of data types.</item> + <tag><c><![CDATA[-Wno_behaviours]]></c>***</tag> + <item>Suppress warnings about behaviour callbacks which drift from the + published recommended interfaces.</item> <tag><c><![CDATA[-Wunmatched_returns]]></c>***</tag> <item>Include warnings for function calls which ignore a structured return value or do not match against one of many possible return @@ -237,9 +243,6 @@ exception.</item> <tag><c><![CDATA[-Wrace_conditions]]></c>***</tag> <item>Include warnings for possible race conditions.</item> - <tag><c><![CDATA[-Wbehaviours]]></c>***</tag> - <item>Include warnings about behaviour callbacks which drift from the - published recommended interfaces.</item> <tag><c><![CDATA[-Wunderspecs]]></c>***</tag> <item>Warn about underspecified functions (the -spec is strictly more allowing than the success typing).</item> diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 3290a98795..cd809662f2 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2012</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,6 +31,34 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 2.6</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Miscellaneous updates due to Unicode support. </p> + <p> + Own Id: OTP-10820</p> + </item> + <item> + <p> + User defined types with same name and different arity and + documentation inconsistencies. Thanks Stavros Aronis.</p> + <p> + Own Id: OTP-10861</p> + </item> + <item> + <p> + Native code compilation changes. Thanks to Kostis + Sagonas.</p> + <p> + Own Id: OTP-10865</p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 2.5.4</title> <section><title>Improvements and New Features</title> diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index 63c51e219a..be4b9b6e12 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -409,9 +409,10 @@ message_to_string({extra_range, [M, F, A, ExtraRanges, SigRange]}) -> io_lib:format("The specification for ~w:~w/~w states that the function" " might also return ~s but the inferred return is ~s\n", [M, F, A, ExtraRanges, SigRange]); -message_to_string({overlapping_contract, []}) -> - "Overloaded contract has overlapping domains;" - " such contracts are currently unsupported and are simply ignored\n"; +message_to_string({overlapping_contract, [M, F, A]}) -> + io_lib:format("Overloaded contract for ~w:~w/~w has overlapping domains;" + " such contracts are currently unsupported and are simply ignored\n", + [M, F, A]); message_to_string({spec_missing_fun, [M, F, A]}) -> io_lib:format("Contract for function that does not exist: ~w:~w/~w\n", [M, F, A]); diff --git a/lib/dialyzer/src/dialyzer_behaviours.erl b/lib/dialyzer/src/dialyzer_behaviours.erl index 36aef2a37f..2b9a69ce77 100644 --- a/lib/dialyzer/src/dialyzer_behaviours.erl +++ b/lib/dialyzer/src/dialyzer_behaviours.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,8 +25,6 @@ %%% %%% Created : 28 Oct 2009 by Stavros Aronis <[email protected]> %%%------------------------------------------------------------------- -%%% NOTE: This module is currently experimental -- do NOT rely on it! -%%%------------------------------------------------------------------- -module(dialyzer_behaviours). @@ -127,15 +125,12 @@ check_all_callbacks(Module, Behaviour, [Cb|Rest], erl_types:t_to_string(CbReturnType, Records)]}|Acc00] end end, - Acc02 = - case erl_types:any_none( - erl_types:t_inf_lists(ArgTypes, CbArgTypes)) of - false -> Acc01; - true -> - find_mismatching_args(type, ArgTypes, CbArgTypes, Behaviour, - Function, Arity, Records, 1, Acc01) - end, - Acc02 + case erl_types:any_none(erl_types:t_inf_lists(ArgTypes, CbArgTypes)) of + false -> Acc01; + true -> + find_mismatching_args(type, ArgTypes, CbArgTypes, Behaviour, + Function, Arity, Records, 1, Acc01) + end end, Acc2 = case dialyzer_codeserver:lookup_mfa_contract(CbMFA, Codeserver) of @@ -157,16 +152,14 @@ check_all_callbacks(Module, Behaviour, [Cb|Rest], erl_types:t_to_string(ExtraType, Records), erl_types:t_to_string(CbReturnType, Records)]}|Acc10] end, - Acc12 = - case erl_types:any_none( - erl_types:t_inf_lists(SpecArgTypes, CbArgTypes)) of - false -> Acc11; - true -> - find_mismatching_args({spec, File, Line}, SpecArgTypes, - CbArgTypes, Behaviour, Function, - Arity, Records, 1, Acc11) - end, - Acc12 + case erl_types:any_none( + erl_types:t_inf_lists(SpecArgTypes, CbArgTypes)) of + false -> Acc11; + true -> + find_mismatching_args({spec, File, Line}, SpecArgTypes, + CbArgTypes, Behaviour, Function, + Arity, Records, 1, Acc11) + end end, NewAcc = Acc2, check_all_callbacks(Module, Behaviour, Rest, State, NewAcc). diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl index 64e0ee88af..b9ad3f857d 100644 --- a/lib/dialyzer/src/dialyzer_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_callgraph.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -798,10 +798,7 @@ condensation(G) -> fun({V1, V2}) -> I1 = ets:lookup_element(V2I, V1, 2), I2 = ets:lookup_element(V2I, V2, 2), - case I1 =:= I2 of - true -> true; - false -> ets:insert(I2I, {I1, I2}) - end + I1 =:= I2 orelse ets:insert(I2I, {I1, I2}) end, lists:foreach(Fun1, digraph:edges(G)), Fun3 = diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 6732d96b98..2456585bd0 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -2,7 +2,7 @@ %%------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -488,6 +488,7 @@ expand_dependent_modules_1([Mod|Mods], Included, ModDeps) -> expand_dependent_modules_1([], Included, _ModDeps) -> Included. +-define(MIN_PARALLELISM, 7). -define(MIN_FILES_FOR_NATIVE_COMPILE, 20). -spec hipe_compile([file:filename()], #options{}) -> 'ok'. @@ -501,11 +502,14 @@ hipe_compile(Files, #options{erlang_mode = ErlangMode} = Options) -> case erlang:system_info(hipe_architecture) of undefined -> ok; _ -> - Mods = [lists, dict, gb_sets, gb_trees, ordsets, sets, + Mods = [lists, dict, digraph, digraph_utils, ets, + gb_sets, gb_trees, ordsets, sets, sofs, cerl, cerl_trees, erl_types, erl_bif_types, - dialyzer_analysis_callgraph, dialyzer_codeserver, - dialyzer_dataflow, dialyzer_dep, dialyzer_plt, - dialyzer_succ_typings, dialyzer_typesig], + dialyzer_analysis_callgraph, dialyzer, dialyzer_behaviours, + dialyzer_codeserver, dialyzer_contracts, + dialyzer_coordinator, dialyzer_dataflow, dialyzer_dep, + dialyzer_plt, dialyzer_succ_typings, dialyzer_typesig, + dialyzer_typesig, dialyzer_worker], report_native_comp(Options), {T1, _} = statistics(wall_clock), native_compile(Mods), @@ -515,12 +519,12 @@ hipe_compile(Files, #options{erlang_mode = ErlangMode} = Options) -> end. native_compile(Mods) -> - case erlang:system_info(schedulers) of - %% N when N > 1 -> - %% Parent = self(), - %% Pids = [spawn(fun () -> Parent ! {self(), hc(M)} end) || M <- Mods], - %% lists:foreach(fun (Pid) -> receive {Pid, Res} -> Res end end, Pids); - _ -> % 1 -> + case dialyzer_utils:parallelism() > ?MIN_PARALLELISM of + true -> + Parent = self(), + Pids = [spawn(fun () -> Parent ! {self(), hc(M)} end) || M <- Mods], + lists:foreach(fun (Pid) -> receive {Pid, Res} -> Res end end, Pids); + false -> lists:foreach(fun (Mod) -> hc(Mod) end, Mods) end. @@ -529,6 +533,7 @@ hc(Mod) -> case code:is_module_native(Mod) of true -> ok; false -> + %% io:format(" ~s", [Mod]), {ok, Mod} = hipe:c(Mod), ok end. diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl index 9989118671..216e06219c 100644 --- a/lib/dialyzer/src/dialyzer_codeserver.erl +++ b/lib/dialyzer/src/dialyzer_codeserver.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2011. All Rights Reserved. +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -100,10 +100,7 @@ ets_dict_to_dict(Table) -> ets:foldl(Fold, dict:new(), Table). ets_set_is_element(Key, Table) -> - case ets:lookup(Table, Key) of - [] -> false; - _ -> true - end. + ets:lookup(Table, Key) =/= []. ets_set_insert_set(Set, Table) -> ets_set_insert_list(sets:to_list(Set), Table). @@ -116,7 +113,7 @@ ets_set_to_set(Table) -> ets:foldl(Fold, sets:new(), Table). ets_read_concurrent_table(Name) -> - ets:new(Name,[{read_concurrency, true}]). + ets:new(Name, [{read_concurrency, true}]). %%-------------------------------------------------------------------- diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 157c951f77..410be8586e 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -520,6 +520,8 @@ get_invalid_contract_warnings_funs([{MFA, {FileLine, Contract}}|Left], case check_contract(Contract, Sig) of {error, invalid_contract} -> [invalid_contract_warning(MFA, FileLine, Sig, RecDict)|Acc]; + {error, {overlapping_contract, []}} -> + [overlapping_contract_warning(MFA, FileLine)|Acc]; {error, {extra_range, ExtraRanges, STRange}} -> Warn = case t_from_forms_without_remote(Contract#contract.forms, @@ -571,6 +573,9 @@ invalid_contract_warning({M, F, A}, FileLine, SuccType, RecDict) -> SuccTypeStr = dialyzer_utils:format_sig(SuccType, RecDict), {?WARN_CONTRACT_TYPES, FileLine, {invalid_contract, [M, F, A, SuccTypeStr]}}. +overlapping_contract_warning({M, F, A}, FileLine) -> + {?WARN_CONTRACT_TYPES, FileLine, {overlapping_contract, [M, F, A]}}. + extra_range_warning({M, F, A}, FileLine, ExtraRanges, STRange) -> ERangesStr = erl_types:t_to_string(ExtraRanges), STRangeStr = erl_types:t_to_string(STRange), diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index 0df003a035..17a292a7d6 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1543,12 +1543,11 @@ get_bif_constr({erlang, '==', 2}, Dst, [Arg1, Arg2] = Args, _State) -> mk_constraint(Arg1, sub, ArgV1), mk_constraint(Arg2, sub, ArgV2)]); get_bif_constr({erlang, element, 2} = _BIF, Dst, Args, - #state{cs = Constrs} = State) -> + #state{cs = Constrs, opaques = Opaques}) -> GenType = erl_bif_types:type(erlang, element, 2), case t_is_none(GenType) of true -> ?debug("Bif: ~w failed\n", [_BIF]), throw(error); false -> - Opaques = State#state.opaques, Fun = fun(Map) -> [I, T] = ATs = lookup_type_list(Args, Map), ATs2 = case lists:member(T, Opaques) of @@ -2582,19 +2581,8 @@ enter_type(Key, Val, Map) when is_integer(Key) -> end end; enter_type(Key, Val, Map) -> - ?debug("Entering ~s :: ~s\n", [format_type(Key), format_type(Val)]), KeyName = t_var_name(Key), - case t_is_any(Val) of - true -> - erase_type(KeyName, Map); - false -> - LimitedVal = t_limit(Val, ?INTERNAL_TYPE_LIMIT), - case dict:find(KeyName, Map) of - {ok, LimitedVal} -> Map; - {ok, _} -> map_store(KeyName, LimitedVal, Map); - error -> map_store(KeyName, LimitedVal, Map) - end - end. + enter_type(KeyName, Val, Map). enter_type_lists([Key|KeyTail], [Val|ValTail], Map) -> Map1 = enter_type(Key, Val, Map), diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index 8046b48838..a4c4d37a0f 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -219,15 +219,18 @@ get_record_and_type_info([], _Module, Records, RecDict) -> end. add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, RecDict) -> - case erl_types:type_is_defined(TypeOrOpaque, Name, RecDict) of + Arity = length(ArgForms), + case erl_types:type_is_defined(TypeOrOpaque, Name, Arity, RecDict) of true -> - throw({error, flat_format("Type ~s already defined\n", [Name])}); + Msg = flat_format("Type ~s/~w already defined\n", [Name, Arity]), + throw({error, Msg}); false -> ArgTypes = [erl_types:t_from_form(X) || X <- ArgForms], case lists:all(fun erl_types:t_is_var/1, ArgTypes) of true -> ArgNames = [erl_types:t_var_name(X) || X <- ArgTypes], - dict:store({TypeOrOpaque, Name}, {Module, TypeForm, ArgNames}, RecDict); + dict:store({TypeOrOpaque, Name, Arity}, + {Module, TypeForm, ArgNames}, RecDict); false -> throw({error, flat_format("Type declaration for ~w does not " "have variables as parameters", [Name])}) diff --git a/lib/dialyzer/test/r9c_SUITE_data/results/inets b/lib/dialyzer/test/r9c_SUITE_data/results/inets index 629378d673..d377f34978 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/results/inets +++ b/lib/dialyzer/test/r9c_SUITE_data/results/inets @@ -1,5 +1,5 @@ -ftp.erl:1243: The pattern {'ok', {N, Bytes}} can never match the type 'eof' | {'error',atom()} | {'ok',binary() | string()} +ftp.erl:1243: The pattern {'ok', {N, Bytes}} can never match the type 'eof' | {'error',atom() | {'no_translation','unicode','latin1'}} | {'ok',binary() | string()} ftp.erl:640: The pattern {'closed', _Why} can never match the type 'perm_fname_not_allowed' | 'perm_neg_compl' | 'perm_no_space' | 'pos_compl' | 'pos_interm' | 'pos_interm_acct' | 'trans_neg_compl' | 'trans_no_space' | {'error' | 'perm_fname_not_allowed' | 'perm_neg_compl' | 'perm_no_space' | 'pos_compl' | 'pos_interm' | 'pos_interm_acct' | 'pos_prel' | 'trans_neg_compl' | 'trans_no_space',atom() | [any()] | {'invalid_server_response',[any(),...]}} http.erl:117: The pattern {'error', Reason} can never match the type #req_headers{connection::[45 | 97 | 101 | 105 | 107 | 108 | 112 | 118,...],content_length::[48,...],other::[{_,_}]} http.erl:138: Function close_session/2 will never be called @@ -34,7 +34,7 @@ mod_auth_plain.erl:100: The variable _ can never match since previous clauses co mod_auth_plain.erl:159: The variable _ can never match since previous clauses completely covered the type [[any()]] mod_auth_plain.erl:83: The variable O can never match since previous clauses completely covered the type [[any()]] mod_cgi.erl:372: The pattern {'http_response', NewAccResponse} can never match the type 'ok' -mod_dir.erl:101: The call lists:flatten(nonempty_improper_list(atom() | [any()] | char(),atom())) will never return since it differs in the 1st argument from the success typing arguments: ([any()]) +mod_dir.erl:101: The call lists:flatten(nonempty_improper_list(atom() | [any()] | char(),atom() | {'no_translation',binary()})) will never return since it differs in the 1st argument from the success typing arguments: ([any()]) mod_dir.erl:72: The pattern {'error', Reason} can never match the type {'ok',[[[any()] | char()],...]} mod_get.erl:135: The pattern <{'enfile', _}, _Info, Path> can never match the type <atom(),#mod{},atom() | binary() | [atom() | [any()] | char()]> mod_head.erl:80: The pattern <{'enfile', _}, _Info, Path> can never match the type <atom(),#mod{},atom() | binary() | [atom() | [any()] | char()]> diff --git a/lib/dialyzer/test/r9c_SUITE_data/results/mnesia b/lib/dialyzer/test/r9c_SUITE_data/results/mnesia index 1aac46f5b2..b73943422a 100644 --- a/lib/dialyzer/test/r9c_SUITE_data/results/mnesia +++ b/lib/dialyzer/test/r9c_SUITE_data/results/mnesia @@ -20,8 +20,8 @@ mnesia_frag_hash.erl:24: Callback info about the mnesia_frag_hash behaviour is n mnesia_frag_old_hash.erl:23: Callback info about the mnesia_frag_hash behaviour is not available mnesia_index.erl:52: The call mnesia_lib:other_val(Var::{_,'commit_work' | 'index' | 'setorbag' | 'storage_type' | {'index',_}},_ReASoN_::any()) will never return since it differs in the 1st argument from the success typing arguments: ({_,'active_replicas' | 'where_to_read' | 'where_to_write'},any()) mnesia_lib.erl:1028: The pattern {'EXIT', Reason} can never match the type [any()] | {'error',_} -mnesia_lib.erl:957: The pattern {'ok', {0, _}} can never match the type 'eof' | {'error',atom()} | {'ok',binary() | string()} -mnesia_lib.erl:959: The pattern {'ok', {_, Bin}} can never match the type 'eof' | {'error',atom()} | {'ok',binary() | string()} +mnesia_lib.erl:957: The pattern {'ok', {0, _}} can never match the type 'eof' | {'error',atom() | {'no_translation','unicode','latin1'}} | {'ok',binary() | string()} +mnesia_lib.erl:959: The pattern {'ok', {_, Bin}} can never match the type 'eof' | {'error',atom() | {'no_translation','unicode','latin1'}} | {'ok',binary() | string()} mnesia_loader.erl:36: The call mnesia_lib:other_val(Var::{_,'access_mode' | 'cstruct' | 'db_nodes' | 'setorbag' | 'snmp' | 'storage_type'},Reason::any()) will never return since it differs in the 1st argument from the success typing arguments: ({_,'active_replicas' | 'where_to_read' | 'where_to_write'},any()) mnesia_locker.erl:1017: Function system_terminate/4 has no local return mnesia_log.erl:707: The test {'error',{[1..255,...],[any(),...]}} | {'ok',_} == atom() can never evaluate to 'true' diff --git a/lib/dialyzer/test/race_SUITE_data/results/extract_translations b/lib/dialyzer/test/race_SUITE_data/results/extract_translations index 62aa1aa511..295404bfed 100644 --- a/lib/dialyzer/test/race_SUITE_data/results/extract_translations +++ b/lib/dialyzer/test/race_SUITE_data/results/extract_translations @@ -1,5 +1,5 @@ -extract_translations.erl:140: The call ets:insert('files',{atom() | binary() | [atom() | [any()] | char()]}) might have an unintended effect due to a possible race condition caused by its combination with the ets:lookup('files',File::atom() | binary() | [atom() | [any()] | char()]) call in extract_translations.erl on line 135 +extract_translations.erl:140: The call ets:insert('files',{atom() | [atom() | [any()] | char()]}) might have an unintended effect due to a possible race condition caused by its combination with the ets:lookup('files',File::atom() | [atom() | [any()] | char()]) call in extract_translations.erl on line 135 extract_translations.erl:146: The call ets:insert('translations',{_,[]}) might have an unintended effect due to a possible race condition caused by its combination with the ets:lookup('translations',Str::any()) call in extract_translations.erl on line 126 -extract_translations.erl:152: The call ets:insert('files',{atom() | binary() | [atom() | [any()] | char()]}) might have an unintended effect due to a possible race condition caused by its combination with the ets:lookup('files',File::atom() | binary() | [atom() | [any()] | char()]) call in extract_translations.erl on line 148 +extract_translations.erl:152: The call ets:insert('files',{atom() | [atom() | [any()] | char()]}) might have an unintended effect due to a possible race condition caused by its combination with the ets:lookup('files',File::atom() | [atom() | [any()] | char()]) call in extract_translations.erl on line 148 extract_translations.erl:154: The call ets:insert('translations',{_,[]}) might have an unintended effect due to a possible race condition caused by its combination with the ets:lookup('translations',Str::any()) call in extract_translations.erl on line 126 diff --git a/lib/dialyzer/test/small_SUITE_data/results/contract3 b/lib/dialyzer/test/small_SUITE_data/results/contract3 index 44b49e745a..6e111f87d9 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/contract3 +++ b/lib/dialyzer/test/small_SUITE_data/results/contract3 @@ -1,3 +1,3 @@ -contract3.erl:17: Overloaded contract has overlapping domains; such contracts are currently unsupported and are simply ignored -contract3.erl:29: Overloaded contract has overlapping domains; such contracts are currently unsupported and are simply ignored +contract3.erl:17: Overloaded contract for contract3:t1/1 has overlapping domains; such contracts are currently unsupported and are simply ignored +contract3.erl:29: Overloaded contract for contract3:t3/3 has overlapping domains; such contracts are currently unsupported and are simply ignored diff --git a/lib/dialyzer/test/small_SUITE_data/results/flatten b/lib/dialyzer/test/small_SUITE_data/results/flatten index 8aa44dd002..0bd866770c 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/flatten +++ b/lib/dialyzer/test/small_SUITE_data/results/flatten @@ -1,2 +1,2 @@ -flatten.erl:17: The call lists:flatten(nonempty_improper_list(atom() | [any()] | char(),atom())) will never return since it differs in the 1st argument from the success typing arguments: ([any()]) +flatten.erl:17: The call lists:flatten(nonempty_improper_list(atom() | [any()] | char(),atom() | {'no_translation',binary()})) will never return since it differs in the 1st argument from the success typing arguments: ([any()]) diff --git a/lib/dialyzer/test/small_SUITE_data/results/types_arity b/lib/dialyzer/test/small_SUITE_data/results/types_arity new file mode 100644 index 0000000000..02641bd167 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/types_arity @@ -0,0 +1,2 @@ + +types_arity.erl:16: Invalid type specification for function types_arity:test2/0. The success typing is () -> {'node','a','nil','nil'} diff --git a/lib/dialyzer/test/small_SUITE_data/src/types_arity.erl b/lib/dialyzer/test/small_SUITE_data/src/types_arity.erl new file mode 100644 index 0000000000..4ddc986ea8 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/types_arity.erl @@ -0,0 +1,20 @@ +-module(types_arity). + +-export([ test1/0 + , test2/0 + , test3/0 + ]). + +-export_type([tree/0, tree/1]). + +-type tree(T) :: 'nil' | {'node', T, tree(T), tree(T)}. +-type tree() :: tree(integer()). + +-spec test1() -> tree(). +test1() -> {node, 7, nil, nil}. + +-spec test2() -> tree(). +test2() -> {node, a, nil, nil}. + +-spec test3() -> tree(atom()). +test3() -> {node, a, nil, nil}. diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index d4c45f3842..bdb8692f8a 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 2.5.4 +DIALYZER_VSN = 2.6 diff --git a/lib/diameter/doc/src/Makefile b/lib/diameter/doc/src/Makefile index 0cbe1f000f..99a6680f12 100644 --- a/lib/diameter/doc/src/Makefile +++ b/lib/diameter/doc/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2012. All Rights Reserved. +# Copyright Ericsson AB 2010-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -35,45 +35,17 @@ XML_REF_FILES = $(XML_REF1_FILES) $(XML_REF3_FILES) $(XML_REF4_FILES) XML_FILES = $(BOOK_FILES) $(XML_APPLICATION_FILES) \ $(XML_REF_FILES) \ $(XML_PART_FILES) $(XML_CHAPTER_FILES) \ - seealso.ent - -INTERNAL_HTML_FILES = $(TECHNICAL_DESCR_FILES:%.xml=$(HTMLDIR)/%.html) - -HTML_APP_FILES = $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) -HTML_EXTRA_FILES = $(XML_EXTRA_FILES:%.xml=$(HTMLDIR)/%.html) -HTML_PART_FILES = $(XML_PART_FILES:%.xml=$(HTMLDIR)/%.html) - -HTML_FILES = $(HTML_APP_FILES) $(HTML_EXTRA_FILES) $(HTML_PART_FILES) + $(XML_EXTRA_FILES) INFO_FILE = ../../info -HTML_REF_FILES = $(XML_REF_FILES:%.xml=$(HTMLDIR)/%.html) - -HTML_CHAPTER_FILES = $(XML_CHAPTER_FILES:%.xml=$(HTMLDIR)/%.html) - -EXTRA_FILES = \ - $(DEFAULT_GIF_FILES) \ - $(DEFAULT_HTML_FILES) \ - $(HTML_REF_FILES) \ - $(HTML_CHAPTER_FILES) - MAN1_FILES = $(XML_REF1_FILES:%.xml=$(MAN1DIR)/%.1) MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) MAN4_FILES = $(XML_REF4_FILES:%.xml=$(MAN4DIR)/%.4) -HTML_REF_MAN_FILE = $(HTMLDIR)/index.html - -TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf STANDARD_DIR = ../standard -STANDARDS = - -# ---------------------------------------------------- -# FLAGS -# ---------------------------------------------------- - -XML_FLAGS += -DVIPS_FLAGS += # ---------------------------------------------------- # Targets @@ -86,11 +58,11 @@ docs: pdf html man ldocs: local_docs $(INDEX_TARGET) -$(TOP_PDF_FILE): $(XML_FILES) +$(PDF_FILE): $(XML_FILES) -pdf: $(TOP_PDF_FILE) +pdf: $(PDF_FILE) -html: gifs $(HTML_REF_MAN_FILE) +html: gifs $(HTMLDIR)/index.html clean clean_docs: clean_pdf clean_html clean_man rm -f errs core *~ @@ -120,7 +92,6 @@ depend: depend.mk debug opt: info: - @echo "->Makefile<-" @echo "" @echo "INDEX_FILE = $(INDEX_FILE)" @echo "INDEX_SRC = $(INDEX_SRC)" @@ -139,13 +110,9 @@ info: @echo "MAN3_FILES = $(MAN3_FILES)" @echo "MAN4_FILES = $(MAN4_FILES)" @echo "" - @echo "HTML_FILES = $(HTML_FILES)" - @echo "TOP_HTML_FILES = $(TOP_HTML_FILES)" - @echo "" @echo "DEFAULT_HTML_FILES = $(DEFAULT_HTML_FILES)" @echo "DEFAULT_GIF_FILES = $(DEFAULT_GIF_FILES)" @echo "" - @echo "" # ---------------------------------------------------- @@ -160,7 +127,7 @@ release_docs_spec: $(LOCAL)docs $(INSTALL_DIR) "$(RELEASE_PATH)/man/man1" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man4" - $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" + $(INSTALL_DATA) $(PDF_FILE) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(HTMLDIR)/*.* "$(RELSYSDIR)/doc/html" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" $(INSTALL_DATA) $(MAN1_FILES) "$(RELEASE_PATH)/man/man1" @@ -173,8 +140,8 @@ release_spec: depend.mk: depend.sed Makefile seealso.ent \ $(XML_REF_FILES) $(XML_CHAPTER_FILES) - $(gen_verbose)sed -f seehere.sed seealso.ent > seehere.ent - $(V_at)(for f in $(XML_REF_FILES) $(XML_CHAPTER_FILES); do \ + sed -f seehere.sed seealso.ent > seehere.ent + (for f in $(XML_REF_FILES) $(XML_CHAPTER_FILES); do \ sed -f $< $$f | sed "s@%FILE%@`basename $$f .xml`@g"; \ done) \ > $@ diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index ba9225da8b..379e9f0738 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -16,7 +16,7 @@ <header> <copyright> -<year>2011</year><year>2012</year> +<year>2011</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -188,7 +188,7 @@ Defaults to the value of the <c>alias</c> option if unspecified.</p> <item> <p> Specifies whether or not the &app_pick_peer; -application callback can modify the application state, +application callback can modify the application state. Defaults to <c>false</c> if unspecified.</p> <note> @@ -206,11 +206,13 @@ probably avoid it.</p> <item> <p> Determines the manner in which incoming answer messages containing -decode errors are handled. +decode errors are handled.</p> + +<p> If <c>callback</c> then errors result in a &app_handle_answer; callback in the same fashion as for &app_handle_request;, with errors communicated in the <c>errors</c> field of the -<c>#diameter_packet{}</c> record passed to the callback. +<c>#diameter_packet{}</c> passed to the callback. If <c>report</c> then an answer containing errors is discarded without a callback and a warning report is written to the log. If <c>discard</c> then an answer containing errors is silently @@ -224,6 +226,39 @@ question is as if a callback had taken place and returned Defaults to <c>report</c> if unspecified.</p> </item> +<tag><c>{request_errors, answer_3xxx|answer|callback}</c></tag> +<item> +<p> +Determines the manner in which incoming requests are handled when an +error other than 3007, DIAMETER_APPLICATION_UNSUPPORTED (which cannot +be associated with an application callback module), is detected.</p> + +<p> +If <c>answer_3xxx</c> then requests are answered without a +&app_handle_request; callback taking place. +If <c>answer</c> then even 5xxx errors are answered without a +callback unless the connection in question has configured the RFC 3588 +common dictionary as noted below. +If <c>callback</c> then a &app_handle_request; callback always takes +place and the return value determines the answer sent to the peer.</p> + +<p> +Defaults to <c>answer_3xxx</c> if unspecified.</p> + +<note> +<p> +Answers sent by diameter set the E-bit in the Diameter Header. +Since RFC 3588 allowed only 3xxx result codes in an +<c>answer-message</c>, <c>answer</c> has the same semantics as +<c>answer_3xxx</c> if the peer connection in question has configured +the RFC 3588 common dictionary, <c>diameter_gen_base_rfc3588</c>. +RFC 6733 allows both 3xxx and 5xxx result codes in an +<c>answer-message</c> so a connection configured with the RFC 6733 +common dictionary, <c>diameter_gen_base_rfc6733</c>, does +distinguish between <c>answer_3xxx</c> and <c>answer</c>.</p> +</note> +</item> + </taglist> <marker id="call_opt"/> @@ -534,7 +569,7 @@ Pkt = #diameter_packet{} The RFC 3539 watchdog state machine has transitioned into (<c>up</c>) or out of (<c>down</c>) the OKAY state. -If a <c>#diameter_packet{}</c> record is present in an <c>up</c> event +If a <c>#diameter_packet{}</c> is present in an <c>up</c> event then there has been a capabilties exchange on a newly established transport connection and the record contains the received CER or CEA. Otherwise a connection has reestablished without the loss or diff --git a/lib/diameter/doc/src/diameter_app.xml b/lib/diameter/doc/src/diameter_app.xml index f4db625c71..d0f1b22ebd 100644 --- a/lib/diameter/doc/src/diameter_app.xml +++ b/lib/diameter/doc/src/diameter_app.xml @@ -13,7 +13,7 @@ <header> <copyright> -<year>2011</year><year>2012</year> +<year>2011</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -475,6 +475,7 @@ not selected.</p> | discard | {eval|eval_packet, Action, PostF}</v> <v>Reply = {reply, &packet; | &message;} + | {answer_message, 3000..3999|5000..5999} | {protocol_error, 3000..3999}</v> <v>Opt = &mod_call_opt;</v> <v>PostF = &mod_evaluable;</v> @@ -509,14 +510,15 @@ Otherwise it contains the record representing the request as outlined in &dict;.</p> <p> -The <c>errors</c> field specifies any Result-Code's identifying errors -that were encountered in decoding the request. -In this case diameter will set both Result-Code and -Failed-AVP AVP's in a returned -answer &message; before sending it to the peer: -the returned &message; need only set any other required AVP's. -Note that the errors detected by diameter are all of the 5xxx series -(Permanent Failures). +The <c>errors</c> field specifies any results codes identifying errors +found while decoding the request. +This is used to set Result-Code and/or Failed-AVP in a returned +answer unless the callback returns a <c>#diameter_packet{}</c> +whose <c>errors</c> field is set to either a non-empty list of its +own, in which case this list is used instead, or the atom <c>false</c> +to disable any setting of Result-Code and Failed-AVP. +Note that the errors detected by diameter are of the 3xxx +and 5xxx series, Protocol Errors and Permanent Failures respectively. The <c>errors</c> list is empty if the request has been received in the relay application.</p> @@ -544,24 +546,25 @@ preserved in the outgoing answer, appropriate values otherwise being set by diameter.</p> </item> -<tag><c>{protocol_error, 3000..3999}</c></tag> +<tag><c>{answer_message, 3000..3999|5000..5999}</c></tag> <item> <p> Send an answer message to the peer containing the specified -protocol error. +Result-Code. Equivalent to</p> <pre> {reply, ['answer-message' | Avps] </pre> <p> where <c>Avps</c> sets the Origin-Host, Origin-Realm, the specified -Result-Code and (if the request sent one) Session-Id AVP's.</p> +Result-Code and (if the request contained one) Session-Id AVP's.</p> <p> -Note that &the_rfc; mandates that only answers with a 3xxx series -Result-Code (protocol errors) may set the E bit. -Returning a non-3xxx value in a <c>protocol_error</c> tuple -will cause the request process in question to fail.</p> +Returning a value other than 3xxx or 5xxx will cause the request +process in question to fail, as will returning a 5xxx value if the +peer connection in question has been configured with the RFC 3588 +common dictionary <c>diameter_gen_base_rfc3588</c>. +(Since RFC 3588 only allows 3xxx values in an answer-message.)</p> </item> <tag><c>{relay, Opts}</c></tag> @@ -614,11 +617,20 @@ containing the encoded binary. The return value is ignored.</p> </item> +<tag><c>{protocol_error, 3000..3999}</c></tag> +<item> +<p> +Equivalent to <c>{answer_message, 3000..3999}</c>.</p> +</item> + </taglist> +<note> <p> -Note that protocol errors detected by diameter will result in an -answer message without <c>handle_request/3</c> being invoked.</p> +Requests containing errors may be answered by diameter, without a +callback taking place, depending on the value of the +&mod_application_opt; <c>request_errors</c>.</p> +</note> </desc> </func> diff --git a/lib/diameter/doc/src/diameter_soc.xml b/lib/diameter/doc/src/diameter_soc.xml index 16f6b9d5bb..4f5419122f 100644 --- a/lib/diameter/doc/src/diameter_soc.xml +++ b/lib/diameter/doc/src/diameter_soc.xml @@ -4,12 +4,12 @@ %also; ]> -<chapter> +<chapter xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> <year>2011</year> -<year>2012</year> +<year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> @@ -80,6 +80,8 @@ just be unnecessary configuration of a value that it has no control over.</p> </list> +<xi:include href="diameter_soc_rfc6733.xml"/> + </section> <!-- ===================================================================== --> diff --git a/lib/diameter/doc/src/diameter_soc_rfc6733.xml b/lib/diameter/doc/src/diameter_soc_rfc6733.xml new file mode 100644 index 0000000000..8d85569650 --- /dev/null +++ b/lib/diameter/doc/src/diameter_soc_rfc6733.xml @@ -0,0 +1,8692 @@ +<?xml version="1.0" encoding="latin1" ?> + +<!-- + +<copyright> +<year>2013</year> +<holder>Ericsson AB. All Rights Reserved.</holder> +</copyright> + +<legalnotice> +The contents of this file are subject to the Erlang Public License, +Version 1.1, (the "License"); you may not use this file except in +compliance with the License. You should have received a copy of the +Erlang Public License along with this software. If not, it can be +retrieved online at http://www.erlang.org/. + +Software distributed under the License is distributed on an "AS IS" +basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +the License for the specific language governing rights and limitations +under the License. + +</legalnotice> + +--> + +<!DOCTYPE section SYSTEM "chapter.dtd" [ + <!ENTITY gen_sctp '<seealso marker="kernel:gen_sctp">gen_sctp(3)</seealso>'> + <!ENTITY gen_tcp '<seealso marker="kernel:gen_tcp">gen_tcp(3)</seealso>'> + <!ENTITY service '<seealso marker="diameter#start_service-2">service</seealso>'> + <!ENTITY capabilities '<seealso marker="diameter#capability">capabilities</seealso>'> + <!ENTITY events '<seealso marker="diameter#service_event">events</seealso>'> + <!ENTITY nada '<p>No comment.</p>'> + <!ENTITY % also SYSTEM "seealso.ent" > + %also; +]> + +<section> +<title>Commentary</title> + +<p> +A more detailed commentary on &the_rfc; follows. +Its purpose is to (hopefully) clarify not only what is supported but +how, given that semantics and features discussed in the RFC are not +solely the responsibility of the diameter application: +in many cases much depends on the configuration a user passes to +diameter, the implementation of &man_app; callback modules in +particular.</p> + +<p> +Comments apply to all text following the preceding comment. +Be sure to distinguish between capitalized <em>Diameter</em>, the +protocol defined by the RFC, and lowercase <em>diameter</em>, the +Erlang application to which the commentary applies.</p> + +<warning> +<p> +The commentary is not yet complete. +Comments currently stop at chapter 4.</p> +</warning> + +<pre> +Fajardo, et al. Standards Track [Page 6] + +RFC 6733 Diameter Base Protocol October 2012 + + +1. Introduction + + Authentication, Authorization, and Accounting (AAA) protocols such as + TACACS [RFC1492] and RADIUS [RFC2865] were initially deployed to + provide dial-up PPP [RFC1661] and terminal server access. Over time, + AAA support was needed on many new access technologies, the scale and + complexity of AAA networks grew, and AAA was also used on new + applications (such as voice over IP). This led to new demands on AAA + protocols. +</pre> + +<p> +Note that diameter implements the Diameter protocol as defined in +&the_rfc;. +It also supported the previous version of the protocol, as defined in +RFC 3588, when there are differences. +(Which will be noted below.) +It does not support RADIUS.</p> + +<pre> + + Network access requirements for AAA protocols are summarized in + Aboba, et al. [RFC2989]. These include: + + Failover + + [RFC2865] does not define failover mechanisms and, as a result, + failover behavior differs between implementations. In order to + provide well-defined failover behavior, Diameter supports + application-layer acknowledgements and defines failover algorithms + and the associated state machine. +</pre> + +&nada; + +<pre> + + Transmission-level security + + RADIUS [RFC2865] defines an application-layer authentication and + integrity scheme that is required only for use with response + packets. While [RFC2869] defines an additional authentication and + integrity mechanism, use is only required during Extensible + Authentication Protocol (EAP) [RFC3748] sessions. While attribute + hiding is supported, [RFC2865] does not provide support for per- + packet confidentiality. In accounting, [RFC2866] assumes that + replay protection is provided by the backend billing server rather + than within the protocol itself. + + While [RFC3162] defines the use of IPsec with RADIUS, support for + IPsec is not required. In order to provide universal support for + transmission-level security, and enable both intra- and inter- + domain AAA deployments, Diameter provides support for TLS/TCP and + DTLS/SCTP. Security is discussed in Section 13. +</pre> + +<p> +Whether or not IPsec is used is transparent to diameter.</p> + +<p> +The transport protocol used on a given peer connection is also +transparent to diameter in that transport to diameter is simply a +module that implements the transport protocol documented in +&man_transport;. +A diameter user configures this module as the &mod_transport_opt; +<c>transport_module</c>.</p> + +<p> +While a user can implement their own transport modules, diameter +includes implementations for TCP and SCTP: +&man_tcp; based on &gen_tcp; and &man_sctp; based on &gen_sctp;. +The former supports TLS but the latter does not currently support +DTLS.</p> + +<pre> + + Reliable transport + + RADIUS runs over UDP, and does not define retransmission behavior; + as a result, reliability varies between implementations. As + described in [RFC2975], this is a major issue in accounting, where + packet loss may translate directly into revenue loss. In order to + + + + + + +Fajardo, et al. Standards Track [Page 7] + +RFC 6733 Diameter Base Protocol October 2012 + + + provide well-defined transport behavior, Diameter runs over + reliable transport mechanisms (TCP, Stream Control Transmission + Protocol (SCTP)) as defined in [RFC3539]. + + Agent support + + RADIUS does not provide for explicit support for agents, including + proxies, redirects, and relays. Since the expected behavior is + not defined, it varies between implementations. Diameter defines + agent behavior explicitly; this is described in Section 2.8. +</pre> + +&nada; + +<pre> + + Server-initiated messages + + While server-initiated messages are defined in RADIUS [RFC5176], + support is optional. This makes it difficult to implement + features such as unsolicited disconnect or re-authentication/ + re-authorization on demand across a heterogeneous deployment. To + address this issue, support for server-initiated messages is + mandatory in Diameter. +</pre> + +<p> +A diameter user can both send and receive messages.</p> + +<pre> + + Transition support + + While Diameter does not share a common protocol data unit (PDU) + with RADIUS, considerable effort has been expended in enabling + backward compatibility with RADIUS so that the two protocols may + be deployed in the same network. Initially, it is expected that + Diameter will be deployed within new network devices, as well as + within gateways enabling communication between legacy RADIUS + devices and Diameter agents. This capability enables Diameter + support to be added to legacy networks, by addition of a gateway + or server speaking both RADIUS and Diameter. +</pre> + +<p> +RADIUS Attributes can be redefined as Diameter AVP's using diameter's +&man_dict; interface but diameter provides no such definitions.</p> + +<pre> + + In addition to addressing the above requirements, Diameter also + provides support for the following: + + Capability negotiation + + RADIUS does not support error messages, capability negotiation, or + a mandatory/non-mandatory flag for attributes. Since RADIUS + clients and servers are not aware of each other's capabilities, + they may not be able to successfully negotiate a mutually + acceptable service or, in some cases, even be aware of what + service has been implemented. Diameter includes support for error + handling (Section 7), capability negotiation (Section 5.3), and + mandatory/non-mandatory Attribute-Value Pairs (AVPs) + (Section 4.1). + + + + + +Fajardo, et al. Standards Track [Page 8] + +RFC 6733 Diameter Base Protocol October 2012 + + + Peer discovery and configuration + + RADIUS implementations typically require that the name or address + of servers or clients be manually configured, along with the + corresponding shared secrets. This results in a large + administrative burden and creates the temptation to reuse the + RADIUS shared secret, which can result in major security + vulnerabilities if the Request Authenticator is not globally and + temporally unique as required in [RFC2865]. Through DNS, Diameter + enables dynamic discovery of peers (see Section 5.2). Derivation + of dynamic session keys is enabled via transmission-level + security. + + Over time, the capabilities of Network Access Server (NAS) devices + have increased substantially. As a result, while Diameter is a + considerably more sophisticated protocol than RADIUS, it remains + feasible to implement it within embedded devices. +</pre> + +&nada; + +<pre> + +1.1. Diameter Protocol + + The Diameter base protocol provides the following facilities: + + o Ability to exchange messages and deliver AVPs +</pre> + +<p> +There are two interfaces directly involved in message exchange when +using diameter: the function &mod_call; for sending outgoing requests, +and the application callback interface, documented in &man_app; for +receiving incoming request and answers.</p> + +<pre> + + o Capabilities negotiation +</pre> + +<p> +Capabilities negotiation is the responsibility of diameter: +a user configures a diameter service and/or transport with +&capabilities; to provide AVP values for CER and CEA messages but it +is diameter itself that sends these messages. +A user receives notification of a successful capabilities exchange by +way of &app_peer_up; callbacks.</p> + +<pre> + + o Error notification +</pre> + +<p> +A user can subscribe to &events;, using &mod_subscribe;, in order to +receive notification of various failures. +Errors in Diameter messaging are communicated via the application +callbacks &app_handle_request;, &app_handle_answer; and +&app_handle_error;.</p> + + +<pre> + + o Extensibility, required in [RFC2989], through addition of new + applications, commands, and AVPs +</pre> + +<p> +Support for applications, commands and AVP's is extensible using +diameter's dictionary interface, as documented in &man_dict;. +Dictionaries are compiled to Erlang encode/decode modules using +&man_compile; or &man_make;.</p> + +<pre> + + o Basic services necessary for applications, such as the handling of + user sessions or accounting +</pre> + +<p> +Compiled dictionaries are provided for the RFC 3588 and RFC 6733 +Diameter applications: common, base accounting and relay. +Dictionaries for a number of standardized +applications are provided in uncompiled form below the <c>examples</c> +subdirectory of the diameter application directory.</p> + +<pre> + + All data delivered by the protocol is in the form of AVPs. Some of + these AVP values are used by the Diameter protocol itself, while + others deliver data associated with particular applications that + employ Diameter. AVPs may be arbitrarily added to Diameter messages, + the only restriction being that the Command Code Format (CCF) + specification (Section 3.2) be satisfied. AVPs are used by the base + Diameter protocol to support the following required features: + + o Transporting of user authentication information, for the purposes + of enabling the Diameter server to authenticate the user + + o Transporting of service-specific authorization information, + between client and servers, allowing the peers to decide whether a + user's access request should be granted + + + +Fajardo, et al. Standards Track [Page 9] + +RFC 6733 Diameter Base Protocol October 2012 + + + o Exchanging resource usage information, which may be used for + accounting purposes, capacity planning, etc. + + o Routing, relaying, proxying, and redirecting of Diameter messages + through a server hierarchy + + The Diameter base protocol satisfies the minimum requirements for a + AAA protocol, as specified by [RFC2989]. The base protocol may be + used by itself for accounting purposes only, or it may be used with a + Diameter application, such as Mobile IPv4 [RFC4004], or network + access [RFC4005]. It is also possible for the base protocol to be + extended for use in new applications, via the addition of new + commands or AVPs. The initial focus of Diameter was network access + and accounting applications. A truly generic AAA protocol used by + many applications might provide functionality not provided by + Diameter. Therefore, it is imperative that the designers of new + applications understand their requirements before using Diameter. + See Section 1.3.4 for more information on Diameter applications. + + Any node can initiate a request. In that sense, Diameter is a peer- + to-peer protocol. In this document, a Diameter client is a device at + the edge of the network that performs access control, such as a + Network Access Server (NAS) or a Foreign Agent (FA). A Diameter + client generates Diameter messages to request authentication, + authorization, and accounting services for the user. A Diameter + agent is a node that does not provide local user authentication or + authorization services; agents include proxies, redirects, and relay + agents. A Diameter server performs authentication and/or + authorization of the user. A Diameter node may act as an agent for + certain requests while acting as a server for others. + + The Diameter protocol also supports server-initiated messages, such + as a request to abort service to a particular user. +</pre> + +&nada; + +<pre> + +1.1.1. Description of the Document Set + + The Diameter specification consists of an updated version of the base + protocol specification (this document) and the Transport Profile + [RFC3539]. This document obsoletes both RFC 3588 and RFC 5719. A + summary of the base protocol updates included in this document can be + found in Section 1.1.3. + + This document defines the base protocol specification for AAA, which + includes support for accounting. There are also a myriad of + applications documents describing applications that use this base + specification for Authentication, Authorization, and Accounting. + These application documents specify how to use the Diameter protocol + within the context of their application. + + + +Fajardo, et al. Standards Track [Page 10] + +RFC 6733 Diameter Base Protocol October 2012 + + + The Transport Profile document [RFC3539] discusses transport layer + issues that arise with AAA protocols and recommendations on how to + overcome these issues. This document also defines the Diameter + failover algorithm and state machine. + + "Clarifications on the Routing of Diameter Request Based on the + Username and the Realm" [RFC5729] defines specific behavior on how to + route requests based on the content of the User-Name AVP (Attribute + Value Pair). + +1.1.2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [RFC2119]. +</pre> + +&nada; + +<pre> + +1.1.3. Changes from RFC 3588 + + This document obsoletes RFC 3588 but is fully backward compatible + with that document. The changes introduced in this document focus on + fixing issues that have surfaced during the implementation of + Diameter (RFC 3588). An overview of some the major changes are given + below. +</pre> + +<p> +RFC 6733 is not fully backwards compatible with RFC 3588. +(For example, in what values of Result-Code values are permissible with +the E-bit.) +The implications of incompatibilities for diameter are noted where +appropriate.</p> + +<pre> + + o Deprecated the use of the Inband-Security AVP for negotiating + Transport Layer Security (TLS) [RFC5246]. It has been generally + considered that bootstrapping of TLS via Inband-Security AVP + creates certain security risks because it does not completely + protect the information carried in the CER/CEA (Capabilities- + Exchange-Request/Capabilities-Exchange-Answer). This version of + Diameter adopts the common approach of defining a well-known + secured port that peers should use when communicating via TLS/TCP + and DTLS/SCTP. This new approach augments the existing in-band + security negotiation, but it does not completely replace it. The + old method is kept for backward compatibility reasons. +</pre> + +<p> +&man_tcp; supports both methods of negotiating TLS: +bootstrapping via Inband-Security and directly following connection +establishment.</p> + +<pre> + + o Deprecated the exchange of CER/CEA messages in the open state. + This feature was implied in the peer state machine table of RFC + 3588, but it was not clearly defined anywhere else in that + document. As work on this document progressed, it became clear + that the multiplicity of meaning and use of Application-Id AVPs in + the CER/CEA messages (and the messages themselves) is seen as an + abuse of the Diameter extensibility rules and thus required + simplification. Capabilities exchange in the open state has been + re-introduced in a separate specification [RFC6737], which clearly + defines new commands for this feature. +</pre> + +<p> +Capabilities exchange in the open state is not supported: an incoming +CER in the open state will cause diameter to ask the relevant +transport process to terminate, which implies the loss of the peer +connection in the case of &man_tcp; and &man_sctp;.</p> + +<p> +Capabilities update, as defined by RFC 6737, is not yet supported. +Support will require diameter to handle CUR/CUA in the same way that +it handles CER/CEA.</p> + +<pre> + + + + + +Fajardo, et al. Standards Track [Page 11] + +RFC 6733 Diameter Base Protocol October 2012 + + + o Simplified security requirements. The use of a secured transport + for exchanging Diameter messages remains mandatory. However, TLS/ + TCP and DTLS/SCTP have become the primary methods of securing + Diameter with IPsec as a secondary alternative. See Section 13 + for details. The support for the End-to-End security framework + (E2E-Sequence AVP and 'P'-bit in the AVP header) has also been + deprecated. +</pre> + +<p> +The End-to-End security framework is not supported since it's use is +largely unspecified: diameter will set the P-bit in outgoing AVP's as +directed by the relevant dictionary and/or &app_prepare_request; or +&app_handle_request; callbacks, but whether or not the P-bit is set on +incoming AVP's has no consequence.</p> + +<p> +As noted above, DTLS is not currently supported and whether or not +IPsec is used is transparent to diameter.</p> + +<pre> + + o Changed Diameter extensibility. This includes fixes to the + Diameter extensibility description (Section 1.3 and others) to + better aid Diameter application designers; in addition, the new + specification relaxes the policy with respect to the allocation of + Command Codes for vendor-specific uses. + + o Clarified Application Id usage. Clarify the proper use of + Application Id information, which can be found in multiple places + within a Diameter message. This includes correlating Application + Ids found in the message headers and AVPs. These changes also + clearly specify the proper Application Id value to use for + specific base protocol messages (ASR/ASA, STR/STA) as well as + clarify the content and use of Vendor-Specific-Application-Id. + + o Clarified routing fixes. This document more clearly specifies + what information (AVPs and Application Ids) can be used for making + general routing decisions. A rule for the prioritization of + redirect routing criteria when multiple route entries are found + via redirects has also been added (see Section 6.13). + + o Simplified Diameter peer discovery. The Diameter discovery + process now supports only widely used discovery schemes; the rest + have been deprecated (see Section 5.2 for details). +</pre> + +<p> +Peer discover is not currently supported: peers to which a node should +connect must be configured. +Connection requests are accepted from arbitrary peers but a +&mod_transport_opt; <c>capabilities_cb</c> can be used to reject a +peer based on an incoming CER or CEA.</p> + +<pre> + + There are many other miscellaneous fixes that have been introduced in + this document that may not be considered significant, but they have + value nonetheless. Examples are removal of obsolete types, fixes to + the state machine, clarification of the election process, message + validation, fixes to Failed-AVP and Result-Code AVP values, etc. All + of the errata filed against RFC 3588 prior to the publication of this + document have been addressed. A comprehensive list of changes is not + shown here for practical reasons. + +1.2. Terminology + + AAA + + Authentication, Authorization, and Accounting. + + + + + +Fajardo, et al. Standards Track [Page 12] + +RFC 6733 Diameter Base Protocol October 2012 + + + ABNF + + Augmented Backus-Naur Form [RFC5234]. A metalanguage with its own + formal syntax and rules. It is based on the Backus-Naur Form and + is used to define message exchanges in a bi-directional + communications protocol. + + Accounting + + The act of collecting information on resource usage for the + purpose of capacity planning, auditing, billing, or cost + allocation. + + Accounting Record + + An accounting record represents a summary of the resource + consumption of a user over the entire session. Accounting servers + creating the accounting record may do so by processing interim + accounting events or accounting events from several devices + serving the same user. + + Authentication + + The act of verifying the identity of an entity (subject). + + Authorization + + The act of determining whether a requesting entity (subject) will + be allowed access to a resource (object). + + Attribute-Value Pair (AVP) + + The Diameter protocol consists of a header followed by one or more + Attribute-Value-Pairs (AVPs). An AVP includes a header and is + used to encapsulate protocol-specific data (e.g., routing + information) as well as authentication, authorization, or + accounting information. +</pre> + +&nada; + +<pre> + + Command Code Format (CCF) + + A modified form of ABNF used to define Diameter commands (see + Section 3.2). +</pre> + +<p> +The <c>@messages</c> section of the &man_dict; format has the CCF as +content.</p> + +<pre> + + Diameter Agent + + A Diameter Agent is a Diameter node that provides relay, proxy, + redirect, or translation services. + + + + +Fajardo, et al. Standards Track [Page 13] + +RFC 6733 Diameter Base Protocol October 2012 + + + Diameter Client + + A Diameter client is a Diameter node that supports Diameter client + applications as well as the base protocol. Diameter clients are + often implemented in devices situated at the edge of a network and + provide access control services for that network. Typical + examples of Diameter clients include the Network Access Server + (NAS) and the Mobile IP Foreign Agent (FA). + + Diameter Node + + A Diameter node is a host process that implements the Diameter + protocol and acts as either a client, an agent, or a server. + + Diameter Peer + + Two Diameter nodes sharing a direct TCP or SCTP transport + connection are called Diameter peers. + + Diameter Server + + A Diameter server is a Diameter node that handles authentication, + authorization, and accounting requests for a particular realm. By + its very nature, a Diameter server must support Diameter server + applications in addition to the base protocol. +</pre> + +<p> +A Diameter Node is implemented by configuring a service +using &mod_start_service; and one or more transports using +&mod_add_transport;. +The service typically represents a Diameter Node but since +capabilities can be configured on individual transports it's more +accurate to say that the node is a collection of transports +advertising the same Origin-Host.</p> + +<p> +The role of a node (agent, client or server) is not something that's +configured explicitly. +Transports are either connecting or listening, depending on whether +diameter should establish a peer connection and send CER or accept +connections and receive CER, but the role a node implements depends +largely on dictionary configuration and &man_app; callback +implementation.</p> + +<pre> + + Downstream + + Downstream is used to identify the direction of a particular + Diameter message from the home server towards the Diameter client. + + Home Realm + + A Home Realm is the administrative domain with which the user + maintains an account relationship. + + Home Server + + A Diameter server that serves the Home Realm. + + Interim Accounting + + An interim accounting message provides a snapshot of usage during + a user's session. Typically, it is implemented in order to + provide for partial accounting of a user's session in case a + device reboot or other network problem prevents the delivery of a + session summary message or session record. + + + + +Fajardo, et al. Standards Track [Page 14] + +RFC 6733 Diameter Base Protocol October 2012 + + + Local Realm + + A local realm is the administrative domain providing services to a + user. An administrative domain may act as a local realm for + certain users while being a home realm for others. + + Multi-session + + A multi-session represents a logical linking of several sessions. + Multi-sessions are tracked by using the Acct-Multi-Session-Id. An + example of a multi-session would be a Multi-link PPP bundle. Each + leg of the bundle would be a session while the entire bundle would + be a multi-session. + + Network Access Identifier + + The Network Access Identifier, or NAI [RFC4282], is used in the + Diameter protocol to extract a user's identity and realm. The + identity is used to identify the user during authentication and/or + authorization while the realm is used for message routing + purposes. + + Proxy Agent or Proxy + + In addition to forwarding requests and responses, proxies make + policy decisions relating to resource usage and provisioning. + Typically, this is accomplished by tracking the state of NAS + devices. While proxies usually do not respond to client requests + prior to receiving a response from the server, they may originate + Reject messages in cases where policies are violated. As a + result, proxies need to understand the semantics of the messages + passing through them, and they may not support all Diameter + applications. + + Realm + + The string in the NAI that immediately follows the '@' character. + NAI realm names are required to be unique and are piggybacked on + the administration of the DNS namespace. Diameter makes use of + the realm, also loosely referred to as domain, to determine + whether messages can be satisfied locally or whether they must be + routed or redirected. In RADIUS, realm names are not necessarily + piggybacked on the DNS namespace but may be independent of it. + + + + + + + + +Fajardo, et al. Standards Track [Page 15] + +RFC 6733 Diameter Base Protocol October 2012 + + + Real-Time Accounting + + Real-time accounting involves the processing of information on + resource usage within a defined time window. Typically, time + constraints are imposed in order to limit financial risk. The + Diameter Credit-Control Application [RFC4006] is an example of an + application that defines real-time accounting functionality. + + Relay Agent or Relay + + Relays forward requests and responses based on routing-related + AVPs and routing table entries. Since relays do not make policy + decisions, they do not examine or alter non-routing AVPs. As a + result, relays never originate messages, do not need to understand + the semantics of messages or non-routing AVPs, and are capable of + handling any Diameter application or message type. Since relays + make decisions based on information in routing AVPs and realm + forwarding tables, they do not keep state on NAS resource usage or + sessions in progress. + + Redirect Agent + + Rather than forwarding requests and responses between clients and + servers, redirect agents refer clients to servers and allow them + to communicate directly. Since redirect agents do not sit in the + forwarding path, they do not alter any AVPs transiting between + client and server. Redirect agents do not originate messages and + are capable of handling any message type, although they may be + configured only to redirect messages of certain types, while + acting as relay or proxy agents for other types. As with relay + agents, redirect agents do not keep state with respect to sessions + or NAS resources. +</pre> + +&nada; + +<pre> + + Session + + A session is a related progression of events devoted to a + particular activity. Diameter application documents provide + guidelines as to when a session begins and ends. All Diameter + packets with the same Session-Id are considered to be part of the + same session. +</pre> + +<p> +Sessions are not something that diameter is aware of. +The function &mod_session_id; can be used to construct appropriate +values for Session-Id AVP's but logic connecting events in the same +session is the responsibility of the diameter user.</p> + +<pre> + + Stateful Agent + + A stateful agent is one that maintains session state information, + by keeping track of all authorized active sessions. Each + authorized session is bound to a particular service, and its state + is considered active either until it is notified otherwise or + until expiration. + + + +Fajardo, et al. Standards Track [Page 16] + +RFC 6733 Diameter Base Protocol October 2012 + + + Sub-session + + A sub-session represents a distinct service (e.g., QoS or data + characteristics) provided to a given session. These services may + happen concurrently (e.g., simultaneous voice and data transfer + during the same session) or serially. These changes in sessions + are tracked with the Accounting-Sub-Session-Id. + + Transaction State + + The Diameter protocol requires that agents maintain transaction + state, which is used for failover purposes. Transaction state + implies that upon forwarding a request, the Hop-by-Hop Identifier + is saved; the field is replaced with a locally unique identifier, + which is restored to its original value when the corresponding + answer is received. The request's state is released upon receipt + of the answer. A stateless agent is one that only maintains + transaction state. + + Translation Agent + + A translation agent (TLA in Figure 4) is a stateful Diameter node + that performs protocol translation between Diameter and another + AAA protocol, such as RADIUS. + + Upstream + + Upstream is used to identify the direction of a particular + Diameter message from the Diameter client towards the home server. + + User + + The entity or device requesting or using some resource, in support + of which a Diameter client has generated a request. +</pre> + +&nada; + +<pre> + +1.3. Approach to Extensibility + + The Diameter protocol is designed to be extensible, using several + mechanisms, including: + + o Defining new AVP values + + o Creating new AVPs + + o Creating new commands + + o Creating new applications + + + + +Fajardo, et al. Standards Track [Page 17] + +RFC 6733 Diameter Base Protocol October 2012 + + + From the point of view of extensibility, Diameter authentication, + authorization, and accounting applications are treated in the same + way. +</pre> + +<p> +Extensibility in diameter is by way of the dictionary interface +documented in &man_dict;: a diameter user creates applications, +commands and AVP's by implementing a new dictionary, +compiling the dictionary to a codec module using &man_compile; or +&man_make;, and configuring the resulting dictionary module on a +service. +The dictionary modules provided with diameter are all implemented in +this manner.</p> + +<pre> + Note: Protocol designers should try to reuse existing functionality, + namely AVP values, AVPs, commands, and Diameter applications. Reuse + simplifies standardization and implementation. To avoid potential + interoperability issues, it is important to ensure that the semantics + of the reused features are well understood. Given that Diameter can + also carry RADIUS attributes as Diameter AVPs, such reuse + considerations also apply to existing RADIUS attributes that may be + useful in a Diameter application. +</pre> + +<p> +Reuse in dictionary files is achieved by way of the <c>@inherits</c> +section. +AVP's are inherited, commands are not.</p> + +<pre> + +1.3.1. Defining New AVP Values + + In order to allocate a new AVP value for AVPs defined in the Diameter + base protocol, the IETF needs to approve a new RFC that describes the + AVP value. IANA considerations for these AVP values are discussed in + Section 11.3. + + The allocation of AVP values for other AVPs is guided by the IANA + considerations of the document that defines those AVPs. Typically, + allocation of new values for an AVP defined in an RFC would require + IETF Review [RFC5226], whereas values for vendor-specific AVPs can be + allocated by the vendor. + +1.3.2. Creating New AVPs + + A new AVP being defined MUST use one of the data types listed in + Sections 4.2 or 4.3. If an appropriate derived data type is already + defined, it SHOULD be used instead of a base data type to encourage + reusability and good design practice. + + In the event that a logical grouping of AVPs is necessary, and + multiple "groups" are possible in a given command, it is recommended + that a Grouped AVP be used (see Section 4.4). + + The creation of new AVPs can happen in various ways. The recommended + approach is to define a new general-purpose AVP in a Standards Track + RFC approved by the IETF. However, as described in Section 11.1.1, + there are other mechanisms. +</pre> + +<p> +Creating new AVP's is an issue for the dictionary designer, not +diameter.</p> + +<pre> + +1.3.3. Creating New Commands + + A new Command Code MUST be allocated when required AVPs (those + indicated as {AVP} in the CCF definition) are added to, deleted from, + or redefined in (for example, by changing a required AVP into an + optional one) an existing command. + + + +Fajardo, et al. Standards Track [Page 18] + +RFC 6733 Diameter Base Protocol October 2012 + + + Furthermore, if the transport characteristics of a command are + changed (for example, with respect to the number of round trips + required), a new Command Code MUST be registered. + + A change to the CCF of a command, such as described above, MUST + result in the definition of a new Command Code. This subsequently + leads to the need to define a new Diameter application for any + application that will use that new command. + + The IANA considerations for Command Codes are discussed in + Section 3.1. +</pre> + +<p> +Creating new commands is an issue for the dictionary designer, not +diameter.</p> + +<pre> + +1.3.4. Creating New Diameter Applications + + Every Diameter application specification MUST have an IANA-assigned + Application Id (see Section 2.4). The managed Application ID space + is flat, and there is no relationship between different Diameter + applications with respect to their Application Ids. As such, there + is no versioning support provided by these Application Ids + themselves; every Diameter application is a standalone application. + If the application has a relationship with other Diameter + applications, such a relationship is not known to Diameter. +</pre> + +<p> +Creating new applications is an issue for the dictionary designer, +not diameter.</p> + +<p> +An application's Application Id is specified in the <c>@id</c> section +of a dictionary file.</p> + +<pre> + + Before describing the rules for creating new Diameter applications, + it is important to discuss the semantics of the AVP occurrences as + stated in the CCF and the M-bit flag (Section 4.1) for an AVP. There + is no relationship imposed between the two; they are set + independently. + + o The CCF indicates what AVPs are placed into a Diameter command by + the sender of that command. Often, since there are multiple modes + of protocol interactions, many of the AVPs are indicated as + optional. + + o The M-bit allows the sender to indicate to the receiver whether or + not understanding the semantics of an AVP and its content is + mandatory. If the M-bit is set by the sender and the receiver + does not understand the AVP or the values carried within that AVP, + then a failure is generated (see Section 7). +</pre> + +<p> +The M-bit is set on outgoing AVP's as directed by the relevant +dictionary. +For incoming AVP's, an M-bit set on an AVP that isn't +explicitly included in the definition of the command in question is +interpreted as a 5001 error, DIAMETER_AVP_UNSUPPORTED, the +consequences of which depend on the value of the &mod_application_opt; +<c>answer_errors</c> or <c>request_errors</c>.</p> + +<pre> + + It is the decision of the protocol designer when to develop a new + Diameter application rather than extending Diameter in other ways. + However, a new Diameter application MUST be created when one or more + of the following criteria are met: + + + + + + + +Fajardo, et al. Standards Track [Page 19] + +RFC 6733 Diameter Base Protocol October 2012 + + + M-bit Setting + + An AVP with the M-bit in the MUST column of the AVP flag table is + added to an existing Command/Application. An AVP with the M-bit + in the MAY column of the AVP flag table is added to an existing + Command/Application. + + Note: The M-bit setting for a given AVP is relevant to an + Application and each command within that application that includes + the AVP. That is, if an AVP appears in two commands for + application Foo and the M-bit settings are different in each + command, then there should be two AVP flag tables describing when + to set the M-bit. + + Commands + + A new command is used within the existing application because + either an additional command is added, an existing command has + been modified so that a new Command Code had to be registered, or + a command has been deleted. + + AVP Flag bits + + If an existing application changes the meaning/semantics of its + AVP Flags or adds new flag bits, then a new Diameter application + MUST be created. + + If the CCF definition of a command allows it, an implementation may + add arbitrary optional AVPs with the M-bit cleared (including vendor- + specific AVPs) to that command without needing to define a new + application. Please refer to Section 11.1.1 for details. +</pre> + +&nada; + +<pre> + +2. Protocol Overview + + The base Diameter protocol concerns itself with establishing + connections to peers, capabilities negotiation, how messages are sent + and routed through peers, and how the connections are eventually torn + down. The base protocol also defines certain rules that apply to all + message exchanges between Diameter nodes. + + Communication between Diameter peers begins with one peer sending a + message to another Diameter peer. The set of AVPs included in the + message is determined by a particular Diameter application. One AVP + that is included to reference a user's session is the Session-Id. + + The initial request for authentication and/or authorization of a user + would include the Session-Id AVP. The Session-Id is then used in all + subsequent messages to identify the user's session (see Section 8 for + + + +Fajardo, et al. Standards Track [Page 20] + +RFC 6733 Diameter Base Protocol October 2012 + + + more information). The communicating party may accept the request or + reject it by returning an answer message with the Result-Code AVP set + to indicate that an error occurred. The specific behavior of the + Diameter server or client receiving a request depends on the Diameter + application employed. + + Session state (associated with a Session-Id) MUST be freed upon + receipt of the Session-Termination-Request, Session-Termination- + Answer, expiration of authorized service time in the Session-Timeout + AVP, and according to rules established in a particular Diameter + application. +</pre> + +<p> +Like Session-Id, session state is maintained by the diameter user: +diameter has no session state of its own and does not interpret +STR/STA in any way.</p> + +<pre> + + The base Diameter protocol may be used by itself for accounting + applications. For authentication and authorization, it is always + extended for a particular application. + + Diameter clients MUST support the base protocol, which includes + accounting. In addition, they MUST fully support each Diameter + application that is needed to implement the client's service, e.g., + Network Access Server Requirements (NASREQ) [RFC2881] and/or Mobile + IPv4. A Diameter client MUST be referred to as "Diameter X Client" + where X is the application that it supports and not a "Diameter + Client". + + Diameter servers MUST support the base protocol, which includes + accounting. In addition, they MUST fully support each Diameter + application that is needed to implement the intended service, e.g., + NASREQ and/or Mobile IPv4. A Diameter server MUST be referred to as + "Diameter X Server" where X is the application that it supports, and + not a "Diameter Server". + + Diameter relays and redirect agents are transparent to the Diameter + applications, but they MUST support the Diameter base protocol, which + includes accounting, and all Diameter applications. + + Diameter proxies MUST support the base protocol, which includes + accounting. In addition, they MUST fully support each Diameter + application that is needed to implement proxied services, e.g., + NASREQ and/or Mobile IPv4. A Diameter proxy MUST be referred to as + "Diameter X Proxy" where X is the application which it supports, and + not a "Diameter Proxy". + +</pre> + +&nada; + +<pre> + + + + + + + + + +Fajardo, et al. Standards Track [Page 21] + +RFC 6733 Diameter Base Protocol October 2012 + + +2.1. Transport + + The Diameter Transport profile is defined in [RFC3539]. + + The base Diameter protocol is run on port 3868 for both TCP [RFC0793] + and SCTP [RFC4960]. For TLS [RFC5246] and Datagram Transport Layer + Security (DTLS) [RFC6347], a Diameter node that initiates a + connection prior to any message exchanges MUST run on port 5658. It + is assumed that TLS is run on top of TCP when it is used, and DTLS is + run on top of SCTP when it is used. +</pre> + +<p> +Which port a transport connects to or listens on is a matter of +configuration. +Both &man_tcp; and &man_sctp; will default to 3868 if no other value +is specified.</p> + +<pre> + + If the Diameter peer does not support receiving TLS/TCP and DTLS/SCTP + connections on port 5658 (i.e., the peer complies only with RFC + 3588), then the initiator MAY revert to using TCP or SCTP on port + 3868. Note that this scheme is kept only for the purpose of backward + compatibility and that there are inherent security vulnerabilities + when the initial CER/CEA messages are sent unprotected (see + Section 5.6). + + Diameter clients MUST support either TCP or SCTP; agents and servers + SHOULD support both. + + A Diameter node MAY initiate connections from a source port other + than the one that it declares it accepts incoming connections on, and + it MUST always be prepared to receive connections on port 3868 for + TCP or SCTP and port 5658 for TLS/TCP and DTLS/SCTP connections. + When DNS-based peer discovery (Section 5.2) is used, the port numbers + received from SRV records take precedence over the default ports + (3868 and 5658). + + A given Diameter instance of the peer state machine MUST NOT use more + than one transport connection to communicate with a given peer, + unless multiple instances exist on the peer, in which, case a + separate connection per process is allowed. +</pre> + +<p> +The &mod_service_opt; <c>restrict_connection</c> controls to what +extent a diameter service allows multiple connections to the same +peer. +(As identified by the value of Origin-Host received from it +during capabilities exchange.)</p> + +<pre> + + When no transport connection exists with a peer, an attempt to + connect SHOULD be made periodically. This behavior is handled via + the Tc timer (see Section 12 for details), whose recommended value is + 30 seconds. There are certain exceptions to this rule, such as when + a peer has terminated the transport connection stating that it does + not wish to communicate. + +</pre> + +<p> +The frequency of reconnection attempts is configured with the +&mod_transport_opt; <c>reconnect_timer</c> and +<c>watchdog_timer</c>.</p> + +<pre> + + When connecting to a peer and either zero or more transports are + specified, TLS SHOULD be tried first, followed by DTLS, then by TCP, + and finally by SCTP. See Section 5.2 for more information on peer + discovery. +</pre> + +<p> +The order in which different transports are attempted depends on the +order of &mod_transport_opt; <c>transport_module</c> and +<c>transport_config</c> tuples in transport configuration.</p> + +<pre> + + + +Fajardo, et al. Standards Track [Page 22] + +RFC 6733 Diameter Base Protocol October 2012 + + + Diameter implementations SHOULD be able to interpret ICMP protocol + port unreachable messages as explicit indications that the server is + not reachable, subject to security policy on trusting such messages. + Further guidance regarding the treatment of ICMP errors can be found + in [RFC5927] and [RFC5461]. Diameter implementations SHOULD also be + able to interpret a reset from the transport and timed-out connection + attempts. If Diameter receives data from the lower layer that cannot + be parsed or identified as a Diameter error made by the peer, the + stream is compromised and cannot be recovered. The transport + connection MUST be closed using a RESET call (send a TCP RST bit) or + an SCTP ABORT message (graceful closure is compromised). +</pre> + +<p> +ICMP messages and other transport-level errors aren't directly +visible to diameter but transport implementations like &man_tcp; and +&man_sctp; propagate these as terminating transport processes.</p> + +<pre> + +2.1.1. SCTP Guidelines + + Diameter messages SHOULD be mapped into SCTP streams in a way that + avoids head-of-the-line (HOL) blocking. Among different ways of + performing the mapping that fulfill this requirement it is + RECOMMENDED that a Diameter node send every Diameter message (request + or response) over stream zero with the unordered flag set. However, + Diameter nodes MAY select and implement other design alternatives for + avoiding HOL blocking such as using multiple streams with the + unordered flag cleared (as originally instructed in RFC 3588). On + the receiving side, a Diameter entity MUST be ready to receive + Diameter messages over any stream, and it is free to return responses + over a different stream. This way, both sides manage the available + streams in the sending direction, independently of the streams chosen + by the other side to send a particular Diameter message. These + messages can be out-of-order and belong to different Diameter + sessions. +</pre> + +<p> +&man_sctp; allows the sender to specify a stream number explicitly. +The stream on which an incoming message is received it passed to +&app_handle_request; and &app_handle_answer; callbacks as +<c>transport_data</c> in a <c>#diameter_packet{}</c>.</p> + +<p> +Ordered or unordered delivery can be configured per transport.</p> + +<pre> + + Out-of-order delivery has special concerns during a connection + establishment and termination. When a connection is established, the + responder side sends a CEA message and moves to R-Open state as + specified in Section 5.6. If an application message is sent shortly + after the CEA and delivered out-of-order, the initiator side, still + in Wait-I-CEA state, will discard the application message and close + the connection. In order to avoid this race condition, the receiver + side SHOULD NOT use out-of-order delivery methods until the first + message has been received from the initiator, proving that it has + moved to I-Open state. To trigger such a message, the receiver side + could send a DWR immediately after sending a CEA. Upon reception of + the corresponding DWA, the receiver side should start using out-of- + order delivery methods to counter the HOL blocking. +</pre> + +<p> +&man_sctp; does not currently allow the user to switch between ordered +and unordered delivery, or to specify the manner of sending per +message: one or the other must be configured, the defaults being +ordered.</p> + +<pre> + + Another race condition may occur when DPR and DPA messages are used. + Both DPR and DPA are small in size; thus, they may be delivered to + the peer faster than application messages when an out-of-order + delivery mechanism is used. Therefore, it is possible that a DPR/DPA + + + +Fajardo, et al. Standards Track [Page 23] + +RFC 6733 Diameter Base Protocol October 2012 + + + exchange completes while application messages are still in transit, + resulting in a loss of these messages. An implementation could + mitigate this race condition, for example, using timers, and wait for + a short period of time for pending application level messages to + arrive before proceeding to disconnect the transport connection. + Eventually, lost messages are handled by the retransmission mechanism + described in Section 5.5.4. + + A Diameter agent SHOULD use dedicated payload protocol identifiers + (PPIDs) for clear text and encrypted SCTP DATA chunks instead of only + using the unspecified payload protocol identifier (value 0). For + this purpose, two PPID values are allocated: the PPID value 46 is for + Diameter messages in clear text SCTP DATA chunks, and the PPID value + 47 is for Diameter messages in protected DTLS/SCTP DATA chunks. +</pre> + +&nada; + +<pre> + +2.2. Securing Diameter Messages + + Connections between Diameter peers SHOULD be protected by TLS/TCP and + DTLS/SCTP. All Diameter base protocol implementations MUST support + the use of TLS/TCP and DTLS/SCTP. If desired, alternative security + mechanisms that are independent of Diameter, such as IPsec [RFC4301], + can be deployed to secure connections between peers. The Diameter + protocol MUST NOT be used without one of TLS, DTLS, or IPsec. +</pre> + +<p> +As noted above, DTLS is not currently supported and IPsec usage is +transparent to diameter. +Security is not enforced by diameter.</p> + +<pre> + +2.3. Diameter Application Compliance + + Application Ids are advertised during the capabilities exchange phase + (see Section 5.3). Advertising support of an application implies + that the sender supports the functionality specified in the + respective Diameter application specification. + + Implementations MAY add arbitrary optional AVPs with the M-bit + cleared (including vendor-specific AVPs) to a command defined in an + application, but only if the command's CCF syntax specification + allows for it. Please refer to Section 11.1.1 for details. +</pre> + +&nada; + +<pre> + +2.4. Application Identifiers + + Each Diameter application MUST have an IANA-assigned Application ID. + The base protocol does not require an Application Id since its + support is mandatory. During the capabilities exchange, Diameter + nodes inform their peers of locally supported applications. + Furthermore, all Diameter messages contain an Application Id, which + is used in the message forwarding process. + + + + + + + +Fajardo, et al. Standards Track [Page 24] + +RFC 6733 Diameter Base Protocol October 2012 + + + The following Application Id values are defined: + + Diameter common message 0 + Diameter base accounting 3 + Relay 0xffffffff +</pre> + +<p> +These applications are implemented in the dictionary modules +<c>diameter_gen_base_rfc6733</c>, <c>diameter_gen_acct_rfc6733</c> and +<c>diameter_relay</c> respectively. +There are also RFC 3588 versions or the common and accounting +dictionaries: <c>diameter_gen_base_rfc3588</c> and +<c>diameter_base_accounting</c>. +(The inconsistent naming is historical.) +Dictionary modules are configured using the &mod_application_opt; +<c>dictionary</c>.</p> + +<pre> + Relay and redirect agents MUST advertise the Relay Application ID, + while all other Diameter nodes MUST advertise locally supported + applications. The receiver of a Capabilities Exchange message + advertising relay service MUST assume that the sender supports all + current and future applications. + + Diameter relay and proxy agents are responsible for finding an + upstream server that supports the application of a particular + message. If none can be found, an error message is returned with the + Result-Code AVP set to DIAMETER_UNABLE_TO_DELIVER. +</pre> + +&nada; + +<pre> + +2.5. Connections vs. Sessions + + This section attempts to provide the reader with an understanding of + the difference between "connection" and "session", which are terms + used extensively throughout this document. + + A connection refers to a transport-level connection between two peers + that is used to send and receive Diameter messages. A session is a + logical concept at the application layer that exists between the + Diameter client and the Diameter server; it is identified via the + Session-Id AVP. + + +--------+ +-------+ +--------+ + | Client | | Relay | | Server | + +--------+ +-------+ +--------+ + <----------> <----------> + peer connection A peer connection B + + <-----------------------------> + User session x + + Figure 1: Diameter Connections and Sessions + + In the example provided in Figure 1, peer connection A is established + between the client and the relay. Peer connection B is established + between the relay and the server. User session X spans from the + client via the relay to the server. Each "user" of a service causes + an auth request to be sent, with a unique session identifier. Once + accepted by the server, both the client and the server are aware of + the session. + + + + +Fajardo, et al. Standards Track [Page 25] + +RFC 6733 Diameter Base Protocol October 2012 + + + It is important to note that there is no relationship between a + connection and a session, and that Diameter messages for multiple + sessions are all multiplexed through a single connection. Also, note + that Diameter messages pertaining to the session, both application- + specific and those that are defined in this document such as ASR/ASA, + RAR/RAA, and STR/STA, MUST carry the Application Id of the + application. Diameter messages pertaining to peer connection + establishment and maintenance such as CER/CEA, DWR/DWA, and DPR/DPA + MUST carry an Application Id of zero (0). +</pre> + +<p> +As noted above, diameter is not involved in session management. +This is the responsibility of the diameter user.</p> + +<pre> + +2.6. Peer Table + + The Diameter peer table is used in message forwarding and is + referenced by the routing table. A peer table entry contains the + following fields: + + Host Identity + + Following the conventions described for the DiameterIdentity- + derived AVP data format in Section 4.3.1, this field contains the + contents of the Origin-Host (Section 6.3) AVP found in the CER or + CEA message. + + StatusT + + This is the state of the peer entry, and it MUST match one of the + values listed in Section 5.6. + + Static or Dynamic + + Specifies whether a peer entry was statically configured or + dynamically discovered. + + Expiration Time + + Specifies the time at which dynamically discovered peer table + entries are to be either refreshed or expired. If public key + certificates are used for Diameter security (e.g., with TLS), this + value MUST NOT be greater than the expiry times in the relevant + certificates. + + TLS/TCP and DTLS/SCTP Enabled + + Specifies whether TLS/TCP and DTLS/SCTP is to be used when + communicating with the peer. + + Additional security information, when needed (e.g., keys, + certificates). +</pre> + +<p> +The Peer Table is not directly accessible to the diameter user. +Information about connected peers can be retrieved using +&mod_service_info;.</p> + +<pre> + + + +Fajardo, et al. Standards Track [Page 26] + +RFC 6733 Diameter Base Protocol October 2012 + + +2.7. Routing Table + + All Realm-Based routing lookups are performed against what is + commonly known as the routing table (see Section 12). Each routing + table entry contains the following fields: + + Realm Name + + This is the field that MUST be used as a primary key in the + routing table lookups. Note that some implementations perform + their lookups based on longest-match-from-the-right on the realm + rather than requiring an exact match. + + Application Identifier + + An application is identified by an Application Id. A route entry + can have a different destination based on the Application Id in + the message header. This field MUST be used as a secondary key + field in routing table lookups. + + Local Action + + The Local Action field is used to identify how a message should be + treated. The following actions are supported: + + 1. LOCAL - Diameter messages that can be satisfied locally and do + not need to be routed to another Diameter entity. + + 2. RELAY - All Diameter messages that fall within this category + MUST be routed to a next-hop Diameter entity that is indicated + by the identifier described below. Routing is done without + modifying any non-routing AVPs. See Section 6.1.9 for + relaying guidelines. + + 3. PROXY - All Diameter messages that fall within this category + MUST be routed to a next Diameter entity that is indicated by + the identifier described below. The local server MAY apply + its local policies to the message by including new AVPs to the + message prior to routing. See Section 6.1.9 for proxying + guidelines. + + 4. REDIRECT - Diameter messages that fall within this category + MUST have the identity of the home Diameter server(s) + appended, and returned to the sender of the message. See + Section 6.1.8 for redirection guidelines. + + + + + + +Fajardo, et al. Standards Track [Page 27] + +RFC 6733 Diameter Base Protocol October 2012 + + + Server Identifier + + The identity of one or more servers to which the message is to be + routed. This identity MUST also be present in the Host Identity + field of the peer table (Section 2.6). When the Local Action is + set to RELAY or PROXY, this field contains the identity of the + server(s) to which the message MUST be routed. When the Local + Action field is set to REDIRECT, this field contains the identity + of one or more servers to which the message MUST be redirected. + + Static or Dynamic + + Specifies whether a route entry was statically configured or + dynamically discovered. + + Expiration Time + + Specifies the time at which a dynamically discovered route table + entry expires. If public key certificates are used for Diameter + security (e.g., with TLS), this value MUST NOT be greater than the + expiry time in the relevant certificates. + + It is important to note that Diameter agents MUST support at least + one of the LOCAL, RELAY, PROXY, or REDIRECT modes of operation. + Agents do not need to support all modes of operation in order to + conform with the protocol specification, but they MUST follow the + protocol compliance guidelines in Section 2. Relay agents and + proxies MUST NOT reorder AVPs. + + The routing table MAY include a default entry that MUST be used for + any requests not matching any of the other entries. The routing + table MAY consist of only such an entry. + + When a request is routed, the target server MUST have advertised the + Application Id (see Section 2.4) for the given message or have + advertised itself as a relay or proxy agent. Otherwise, an error is + returned with the Result-Code AVP set to DIAMETER_UNABLE_TO_DELIVER. +</pre> + +<p> +Routing does not need specific support in diameter: a user can +maintain their own routing table if desired and implement any desired +routing in &man_app; callbacks. +However, it may be convenient to add more specific routing support to +diameter in the future.</p> + +<pre> + +2.8. Role of Diameter Agents + + In addition to clients and servers, the Diameter protocol introduces + relay, proxy, redirect, and translation agents, each of which is + defined in Section 1.2. Diameter agents are useful for several + reasons: +</pre> + +<p> +An noted above, the role a node plays is largely a question of +configuration and &man_app; callback implementation.</p> + +<pre> + + o They can distribute administration of systems to a configurable + grouping, including the maintenance of security associations. + + + + +Fajardo, et al. Standards Track [Page 28] + +RFC 6733 Diameter Base Protocol October 2012 + + + o They can be used for concentration of requests from a number of + co-located or distributed NAS equipment sets to a set of like user + groups. + + o They can do value-added processing to the requests or responses. + + o They can be used for load balancing. + + o A complex network will have multiple authentication sources, they + can sort requests and forward towards the correct target. + + The Diameter protocol requires that agents maintain transaction + state, which is used for failover purposes. Transaction state + implies that upon forwarding a request, its Hop-by-Hop Identifier is + saved; the field is replaced with a locally unique identifier, which + is restored to its original value when the corresponding answer is + received. The request's state is released upon receipt of the + answer. A stateless agent is one that only maintains transaction + state. + + The Proxy-Info AVP allows stateless agents to add local state to a + Diameter request, with the guarantee that the same state will be + present in the answer. However, the protocol's failover procedures + require that agents maintain a copy of pending requests. + + A stateful agent is one that maintains session state information by + keeping track of all authorized active sessions. Each authorized + session is bound to a particular service, and its state is considered + active until either the agent is notified otherwise or the session + expires. Each authorized session has an expiration, which is + communicated by Diameter servers via the Session-Timeout AVP. + + Maintaining session state may be useful in certain applications, such + as: + + o Protocol translation (e.g., RADIUS <-> Diameter) + + o Limiting resources authorized to a particular user + + o Per-user or per-transaction auditing + + A Diameter agent MAY act in a stateful manner for some requests and + be stateless for others. A Diameter implementation MAY act as one + type of agent for some requests and as another type of agent for + others. +</pre> + +&nada; + +<pre> + + + + + + +Fajardo, et al. Standards Track [Page 29] + +RFC 6733 Diameter Base Protocol October 2012 + + +2.8.1. Relay Agents + + Relay agents are Diameter agents that accept requests and route + messages to other Diameter nodes based on information found in the + messages (e.g., the value of the Destination-Realm AVP Section 6.6). + This routing decision is performed using a list of supported realms + and known peers. This is known as the routing table, as is defined + further in Section 2.7. + + Relays may, for example, be used to aggregate requests from multiple + Network Access Servers (NASes) within a common geographical area + (Point of Presence, POP). The use of relays is advantageous since it + eliminates the need for NASes to be configured with the necessary + security information they would otherwise require to communicate with + Diameter servers in other realms. Likewise, this reduces the + configuration load on Diameter servers that would otherwise be + necessary when NASes are added, changed, or deleted. + + Relays modify Diameter messages by inserting and removing routing + information, but they do not modify any other portion of a message. + Relays SHOULD NOT maintain session state but MUST maintain + transaction state. + + +------+ ---------> +------+ ---------> +------+ + | | 1. Request | | 2. Request | | + | NAS | | DRL | | HMS | + | | 4. Answer | | 3. Answer | | + +------+ <--------- +------+ <--------- +------+ + example.net example.net example.com + + Figure 2: Relaying of Diameter messages + + The example provided in Figure 2 depicts a request issued from a NAS, + which is an access device, for the user [email protected]. Prior to + issuing the request, the NAS performs a Diameter route lookup, using + "example.com" as the key, and determines that the message is to be + relayed to a DRL, which is a Diameter relay. The DRL performs the + same route lookup as the NAS, and relays the message to the HMS, + which is example.com's home server. The HMS identifies that the + request can be locally supported (via the realm), processes the + authentication and/or authorization request, and replies with an + answer, which is routed back to the NAS using saved transaction + state. + + Since relays do not perform any application-level processing, they + provide relaying services for all Diameter applications; therefore, + they MUST advertise the Relay Application Id. +</pre> + +<p> +Requests are relayed by returning a <c>relay</c> tuple from a +&app_handle_request; callback.</p> + +<pre> + + + +Fajardo, et al. Standards Track [Page 30] + +RFC 6733 Diameter Base Protocol October 2012 + + +2.8.2. Proxy Agents + + Similar to relays, proxy agents route Diameter messages using the + Diameter routing table. However, they differ since they modify + messages to implement policy enforcement. This requires that proxies + maintain the state of their downstream peers (e.g., access devices) + to enforce resource usage, provide admission control, and provide + provisioning. + + Proxies may, for example, be used in call control centers or access + ISPs that provide outsourced connections; they can monitor the number + and type of ports in use and make allocation and admission decisions + according to their configuration. + + Since enforcing policies requires an understanding of the service + being provided, proxies MUST only advertise the Diameter applications + they support. +</pre> + +&nada; + +<pre> + +2.8.3. Redirect Agents + + Redirect agents are useful in scenarios where the Diameter routing + configuration needs to be centralized. An example is a redirect + agent that provides services to all members of a consortium, but does + not wish to be burdened with relaying all messages between realms. + This scenario is advantageous since it does not require that the + consortium provide routing updates to its members when changes are + made to a member's infrastructure. + + Since redirect agents do not relay messages, and only return an + answer with the information necessary for Diameter agents to + communicate directly, they do not modify messages. Since redirect + agents do not receive answer messages, they cannot maintain session + state. + + The example provided in Figure 3 depicts a request issued from the + access device, NAS, for the user [email protected]. The message is + forwarded by the NAS to its relay, DRL, which does not have a routing + entry in its Diameter routing table for example.com. The DRL has a + default route configured to DRD, which is a redirect agent that + returns a redirect notification to DRL, as well as the HMS' contact + information. Upon receipt of the redirect notification, the DRL + establishes a transport connection with the HMS, if one doesn't + already exist, and forwards the request to it. + + + + + + + + +Fajardo, et al. Standards Track [Page 31] + +RFC 6733 Diameter Base Protocol October 2012 + + + +------+ + | | + | DRD | + | | + +------+ + ^ | + 2. Request | | 3. Redirection + | | Notification + | v + +------+ ---------> +------+ ---------> +------+ + | | 1. Request | | 4. Request | | + | NAS | | DRL | | HMS | + | | 6. Answer | | 5. Answer | | + +------+ <--------- +------+ <--------- +------+ + example.net example.net example.com + + Figure 3: Redirecting a Diameter Message + + Since redirect agents do not perform any application-level + processing, they provide relaying services for all Diameter + applications; therefore, they MUST advertise the Relay Application + ID. +</pre> + +&nada; + +<pre> + +2.8.4. Translation Agents + + A translation agent is a device that provides translation between two + protocols (e.g., RADIUS<->Diameter, TACACS+<->Diameter). Translation + agents are likely to be used as aggregation servers to communicate + with a Diameter infrastructure, while allowing for the embedded + systems to be migrated at a slower pace. + + Given that the Diameter protocol introduces the concept of long-lived + authorized sessions, translation agents MUST be session stateful and + MUST maintain transaction state. + + Translation of messages can only occur if the agent recognizes the + application of a particular request; therefore, translation agents + MUST only advertise their locally supported applications. + + +------+ ---------> +------+ ---------> +------+ + | | RADIUS Request | | Diameter Request | | + | NAS | | TLA | | HMS | + | | RADIUS Answer | | Diameter Answer | | + +------+ <--------- +------+ <--------- +------+ + example.net example.net example.com + + Figure 4: Translation of RADIUS to Diameter +</pre> + +&nada; + +<pre> + + + + +Fajardo, et al. Standards Track [Page 32] + +RFC 6733 Diameter Base Protocol October 2012 + + +2.9. Diameter Path Authorization + + As noted in Section 2.2, Diameter provides transmission-level + security for each connection using TLS/TCP and DTLS/SCTP. Therefore, + each connection can be authenticated and can be replay and integrity + protected. + + In addition to authenticating each connection, the entire session + MUST also be authorized. Before initiating a connection, a Diameter + peer MUST check that its peers are authorized to act in their roles. + For example, a Diameter peer may be authentic, but that does not mean + that it is authorized to act as a Diameter server advertising a set + of Diameter applications. + + Prior to bringing up a connection, authorization checks are performed + at each connection along the path. Diameter capabilities negotiation + (CER/CEA) also MUST be carried out, in order to determine what + Diameter applications are supported by each peer. Diameter sessions + MUST be routed only through authorized nodes that have advertised + support for the Diameter application required by the session. + + As noted in Section 6.1.9, a relay or proxy agent MUST append a + Route-Record AVP to all requests forwarded. The AVP contains the + identity of the peer from which the request was received. + + The home Diameter server, prior to authorizing a session, MUST check + the Route-Record AVPs to make sure that the route traversed by the + request is acceptable. For example, administrators within the home + realm may not wish to honor requests that have been routed through an + untrusted realm. By authorizing a request, the home Diameter server + is implicitly indicating its willingness to engage in the business + transaction as specified by any contractual relationship between the + server and the previous hop. A DIAMETER_AUTHORIZATION_REJECTED error + message (see Section 7.1.5) is sent if the route traversed by the + request is unacceptable. + + A home realm may also wish to check that each accounting request + message corresponds to a Diameter response authorizing the session. + Accounting requests without corresponding authorization responses + SHOULD be subjected to further scrutiny, as should accounting + requests indicating a difference between the requested and provided + service. + + Forwarding of an authorization response is considered evidence of a + willingness to take on financial risk relative to the session. A + local realm may wish to limit this exposure, for example, by + establishing credit limits for intermediate realms and refusing to + accept responses that would violate those limits. By issuing an + + + +Fajardo, et al. Standards Track [Page 33] + +RFC 6733 Diameter Base Protocol October 2012 + + + accounting request corresponding to the authorization response, the + local realm implicitly indicates its agreement to provide the service + indicated in the authorization response. If the service cannot be + provided by the local realm, then a DIAMETER_UNABLE_TO_COMPLY error + message MUST be sent within the accounting request; a Diameter client + receiving an authorization response for a service that it cannot + perform MUST NOT substitute an alternate service and then send + accounting requests for the alternate service instead. +</pre> + +&nada; + +<pre> + +3. Diameter Header + + A summary of the Diameter header format is shown below. The fields + are transmitted in network byte order. + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Version | Message Length | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Command Flags | Command Code | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Application-ID | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Hop-by-Hop Identifier | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | End-to-End Identifier | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | AVPs ... + +-+-+-+-+-+-+-+-+-+-+-+-+- +</pre> + +<p> +The Diameter Header is represented by the <c>diameter_header</c> +record defined in <c>diameter.hrl</c>. +The <c>diameter_packet</c> record contains a <c>header</c> field whose +value will be a decoded <c>#diameter_header{}</c> for incoming +messages passed to &app_handle_request; and &app_handle_answer; +callbacks. +In the case of outgoing messages, diameter and the relevant +dictionary populate the Diameter Header appropriately, although +&app_prepare_request; and &app_handle_request; callbacks can modify +header values. +(Which can be useful in test.)</p> + +<pre> + + Version + + This Version field MUST be set to 1 to indicate Diameter Version + 1. + + Message Length + + The Message Length field is three octets and indicates the length + of the Diameter message including the header fields and the padded + AVPs. Thus, the Message Length field is always a multiple of 4. + + Command Flags + + The Command Flags field is eight bits. The following bits are + assigned: + + + + + + +Fajardo, et al. Standards Track [Page 34] + +RFC 6733 Diameter Base Protocol October 2012 + + + 0 1 2 3 4 5 6 7 + +-+-+-+-+-+-+-+-+ + |R P E T r r r r| + +-+-+-+-+-+-+-+-+ + + R(equest) + + If set, the message is a request. If cleared, the message is + an answer. + + P(roxiable) + + If set, the message MAY be proxied, relayed, or redirected. If + cleared, the message MUST be locally processed. + + E(rror) + + If set, the message contains a protocol error, and the message + will not conform to the CCF described for this command. + Messages with the 'E' bit set are commonly referred to as error + messages. This bit MUST NOT be set in request messages (see + Section 7.2). + + T(Potentially retransmitted message) + + This flag is set after a link failover procedure, to aid the + removal of duplicate requests. It is set when resending + requests not yet acknowledged, as an indication of a possible + duplicate due to a link failure. This bit MUST be cleared when + sending a request for the first time; otherwise, the sender + MUST set this flag. Diameter agents only need to be concerned + about the number of requests they send based on a single + received request; retransmissions by other entities need not be + tracked. Diameter agents that receive a request with the T + flag set, MUST keep the T flag set in the forwarded request. + This flag MUST NOT be set if an error answer message (e.g., a + protocol error) has been received for the earlier message. It + can be set only in cases where no answer has been received from + the server for a request, and the request has been sent again. + This flag MUST NOT be set in answer messages. + + r(eserved) + + These flag bits are reserved for future use; they MUST be set + to zero and ignored by the receiver. +</pre> + +<p> +Reserved bits are set to 0 in outgoing messages.</p> + +<pre> + + + + + + +Fajardo, et al. Standards Track [Page 35] + +RFC 6733 Diameter Base Protocol October 2012 + + + Command Code + + The Command Code field is three octets and is used in order to + communicate the command associated with the message. The 24-bit + address space is managed by IANA (see Section 3.1). Command Code + values 16,777,214 and 16,777,215 (hexadecimal values FFFFFE- + FFFFFF) are reserved for experimental use (see Section 11.2). + + Application-ID + + Application-ID is four octets and is used to identify for which + application the message is applicable. The application can be an + authentication application, an accounting application, or a + vendor-specific application. + + The value of the Application-ID field in the header MUST be the + same as any relevant Application-Id AVPs contained in the message. + + Hop-by-Hop Identifier + + The Hop-by-Hop Identifier is an unsigned 32-bit integer field (in + network byte order) that aids in matching requests and replies. + The sender MUST ensure that the Hop-by-Hop Identifier in a request + is unique on a given connection at any given time, and it MAY + attempt to ensure that the number is unique across reboots. The + sender of an answer message MUST ensure that the Hop-by-Hop + Identifier field contains the same value that was found in the + corresponding request. The Hop-by-Hop Identifier is normally a + monotonically increasing number, whose start value was randomly + generated. An answer message that is received with an unknown + Hop-by-Hop Identifier MUST be discarded. + + End-to-End Identifier + + The End-to-End Identifier is an unsigned 32-bit integer field (in + network byte order) that is used to detect duplicate messages. + Upon reboot, implementations MAY set the high order 12 bits to + contain the low order 12 bits of current time, and the low order + 20 bits to a random value. Senders of request messages MUST + insert a unique identifier on each message. The identifier MUST + remain locally unique for a period of at least 4 minutes, even + across reboots. The originator of an answer message MUST ensure + that the End-to-End Identifier field contains the same value that + was found in the corresponding request. The End-to-End Identifier + MUST NOT be modified by Diameter agents of any kind. The + combination of the Origin-Host AVP (Section 6.3) and this field is + used to detect duplicates. Duplicate requests SHOULD cause the + same answer to be transmitted (modulo the Hop-by-Hop Identifier + + + +Fajardo, et al. Standards Track [Page 36] + +RFC 6733 Diameter Base Protocol October 2012 + + + field and any routing AVPs that may be present), and they MUST NOT + affect any state that was set when the original request was + processed. Duplicate answer messages that are to be locally + consumed (see Section 6.2) SHOULD be silently discarded. + + AVPs + + AVPs are a method of encapsulating information relevant to the + Diameter message. See Section 4 for more information on AVPs. +</pre> + +&nada; + +<pre> + +3.1. Command Codes + + Each command Request/Answer pair is assigned a Command Code, and the + sub-type (i.e., request or answer) is identified via the 'R' bit in + the Command Flags field of the Diameter header. + + Every Diameter message MUST contain a Command Code in its header's + Command Code field, which is used to determine the action that is to + be taken for a particular message. The following Command Codes are + defined in the Diameter base protocol: + + Section + Command Name Abbrev. Code Reference + -------------------------------------------------------- + Abort-Session-Request ASR 274 8.5.1 + Abort-Session-Answer ASA 274 8.5.2 + Accounting-Request ACR 271 9.7.1 + Accounting-Answer ACA 271 9.7.2 + Capabilities-Exchange- CER 257 5.3.1 + Request + Capabilities-Exchange- CEA 257 5.3.2 + Answer + Device-Watchdog-Request DWR 280 5.5.1 + Device-Watchdog-Answer DWA 280 5.5.2 + Disconnect-Peer-Request DPR 282 5.4.1 + Disconnect-Peer-Answer DPA 282 5.4.2 + Re-Auth-Request RAR 258 8.3.1 + Re-Auth-Answer RAA 258 8.3.2 + Session-Termination- STR 275 8.4.1 + Request + Session-Termination- STA 275 8.4.2 + Answer +</pre> + +<p> +These messages are all defined in diameter's implementation of the +common dictionary in modules <c>diameter_gen_base_rfc6733</c> and +<c>diameter_gen_base_rfc3588</c>. +Corresponding record definitions are found in +<c>diameter_gen_base_rfc6733.hrl</c> and +<c>diameter_gen_base_rfc3588.hrl</c>.</p> + +<pre> + + + + + + + + + +Fajardo, et al. Standards Track [Page 37] + +RFC 6733 Diameter Base Protocol October 2012 + + +3.2. Command Code Format Specification + + Every Command Code defined MUST include a corresponding Command Code + Format (CCF) specification, which is used to define the AVPs that + MUST or MAY be present when sending the message. The following ABNF + specifies the CCF used in the definition: +</pre> + +<p> +The CCF is what is specified in the <c>@messages</c> section of the +&man_dict; format, except as noted below.</p> + +<pre> + + command-def = "<" command-name ">" "::=" diameter-message +</pre> + +<p> +Angle brackets are currently not allowed here. +This was a change between RFC 3588 and RFC 6733: the former disallowed +them in the grammar but included them in its own command definitions.</p> + +<pre> + + command-name = diameter-name + + diameter-name = ALPHA *(ALPHA / DIGIT / "-") + + diameter-message = header *fixed *required *optional + + header = "<Diameter-Header:" command-id + [r-bit] [p-bit] [e-bit] [application-id]">" + + application-id = 1*DIGIT + + command-id = 1*DIGIT + ; The Command Code assigned to the command. + + r-bit = ", REQ" + ; If present, the 'R' bit in the Command + ; Flags is set, indicating that the message + ; is a request as opposed to an answer. + + p-bit = ", PXY" + ; If present, the 'P' bit in the Command + ; Flags is set, indicating that the message + ; is proxiable. + + e-bit = ", ERR" + ; If present, the 'E' bit in the Command + ; Flags is set, indicating that the answer + ; message contains a Result-Code AVP in + ; the "protocol error" class. + + fixed = [qual] "<" avp-spec ">" + ; Defines the fixed position of an AVP. + + required = [qual] "{" avp-spec "}" + ; The AVP MUST be present and can appear + ; anywhere in the message. + + + + + + +Fajardo, et al. Standards Track [Page 38] + +RFC 6733 Diameter Base Protocol October 2012 + + + optional = [qual] "[" avp-name "]" + ; The avp-name in the 'optional' rule cannot + ; evaluate to any AVP Name that is included + ; in a fixed or required rule. The AVP can + ; appear anywhere in the message. + ; + ; NOTE: "[" and "]" have a slightly different + ; meaning than in ABNF. These braces + ; cannot be used to express optional fixed rules + ; (such as an optional ICV at the end). To do + ; this, the convention is '0*1fixed'. + + qual = [min] "*" [max] + ; See ABNF conventions, RFC 5234, Section 4. + ; The absence of any qualifier depends on + ; whether it precedes a fixed, required, or + ; optional rule. If a fixed or required rule has + ; no qualifier, then exactly one such AVP MUST + ; be present. If an optional rule has no + ; qualifier, then 0 or 1 such AVP may be + ; present. If an optional rule has a qualifier, + ; then the value of min MUST be 0 if present. + + min = 1*DIGIT + ; The minimum number of times the element may + ; be present. If absent, the default value is 0 + ; for fixed and optional rules and 1 for + ; required rules. The value MUST be at least 1 + ; for required rules. + + max = 1*DIGIT + ; The maximum number of times the element may + ; be present. If absent, the default value is + ; infinity. A value of 0 implies the AVP MUST + ; NOT be present. + + avp-spec = diameter-name + ; The avp-spec has to be an AVP Name, defined + ; in the base or extended Diameter + ; specifications. + + avp-name = avp-spec / "AVP" + ; The string "AVP" stands for *any* arbitrary AVP + ; Name, not otherwise listed in that Command Code + ; definition. The inclusion of this string + ; is recommended for all CCFs to allow for + ; extensibility. + + + + +Fajardo, et al. Standards Track [Page 39] + +RFC 6733 Diameter Base Protocol October 2012 + + + The following is a definition of a fictitious Command Code: + + Example-Request ::= < Diameter Header: 9999999, REQ, PXY > + { User-Name } + 1* { Origin-Host } + * [ AVP ] +</pre> + +&nada; + +<pre> + +3.3. Diameter Command Naming Conventions + + Diameter command names typically includes one or more English words + followed by the verb "Request" or "Answer". Each English word is + delimited by a hyphen. A three-letter acronym for both the request + and answer is also normally provided. + + An example is a message set used to terminate a session. The command + name is Session-Terminate-Request and Session-Terminate-Answer, while + the acronyms are STR and STA, respectively. + + Both the request and the answer for a given command share the same + Command Code. The request is identified by the R(equest) bit in the + Diameter header set to one (1), to ask that a particular action be + performed, such as authorizing a user or terminating a session. Once + the receiver has completed the request, it issues the corresponding + answer, which includes a result code that communicates one of the + following: + + o The request was successful + + o The request failed + + o An additional request has to be sent to provide information the + peer requires prior to returning a successful or failed answer. + + o The receiver could not process the request, but provides + information about a Diameter peer that is able to satisfy the + request, known as redirect. + + Additional information, encoded within AVPs, may also be included in + answer messages. +</pre> + +<p> +The &man_dict; format places no requirement on the naming of commands.</p> + +<pre> + +4. Diameter AVPs + + Diameter AVPs carry specific authentication, accounting, + authorization, and routing information as well as configuration + details for the request and reply. + + + + + + +Fajardo, et al. Standards Track [Page 40] + +RFC 6733 Diameter Base Protocol October 2012 + + + Each AVP of type OctetString MUST be padded to align on a 32-bit + boundary, while other AVP types align naturally. A number of zero- + valued bytes are added to the end of the AVP Data field until a word + boundary is reached. The length of the padding is not reflected in + the AVP Length field. + +4.1. AVP Header + + The fields in the AVP header MUST be sent in network byte order. The + format of the header is: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | AVP Code | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |V M P r r r r r| AVP Length | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Vendor-ID (opt) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Data ... + +-+-+-+-+-+-+-+-+ + + AVP Code + + The AVP Code, combined with the Vendor-Id field, identifies the + attribute uniquely. AVP numbers 1 through 255 are reserved for + reuse of RADIUS attributes, without setting the Vendor-Id field. + AVP numbers 256 and above are used for Diameter, which are + allocated by IANA (see Section 11.1.1). + + AVP Flags + + The AVP Flags field informs the receiver how each attribute must + be handled. New Diameter applications SHOULD NOT define + additional AVP Flag bits. However, note that new Diameter + applications MAY define additional bits within the AVP header, and + an unrecognized bit SHOULD be considered an error. The sender of + the AVP MUST set 'R' (reserved) bits to 0 and the receiver SHOULD + ignore all 'R' (reserved) bits. The 'P' bit has been reserved for + future usage of end-to-end security. At the time of writing, + there are no end-to-end security mechanisms specified; therefore, + the 'P' bit SHOULD be set to 0. + + The 'M' bit, known as the Mandatory bit, indicates whether the + receiver of the AVP MUST parse and understand the semantics of the + AVP including its content. The receiving entity MUST return an + appropriate error message if it receives an AVP that has the M-bit + + + +Fajardo, et al. Standards Track [Page 41] + +RFC 6733 Diameter Base Protocol October 2012 + + + set but does not understand it. An exception applies when the AVP + is embedded within a Grouped AVP. See Section 4.4 for details. + Diameter relay and redirect agents MUST NOT reject messages with + unrecognized AVPs. + + The 'M' bit MUST be set according to the rules defined in the + application specification that introduces or reuses this AVP. + Within a given application, the M-bit setting for an AVP is + defined either for all command types or for each command type. + + AVPs with the 'M' bit cleared are informational only; a receiver + that receives a message with such an AVP that is not supported, or + whose value is not supported, MAY simply ignore the AVP. + + The 'V' bit, known as the Vendor-Specific bit, indicates whether + the optional Vendor-ID field is present in the AVP header. When + set, the AVP Code belongs to the specific vendor code address + space. + + AVP Length + + The AVP Length field is three octets, and indicates the number of + octets in this AVP including the AVP Code field, AVP Length field, + AVP Flags field, Vendor-ID field (if present), and the AVP Data + field. If a message is received with an invalid attribute length, + the message MUST be rejected. + +4.1.1. Optional Header Elements + + The AVP header contains one optional field. This field is only + present if the respective bit-flag is enabled. + + Vendor-ID + + The Vendor-ID field is present if the 'V' bit is set in the AVP + Flags field. The optional four-octet Vendor-ID field contains the + IANA-assigned "SMI Network Management Private Enterprise Codes" + [ENTERPRISE] value, encoded in network byte order. Any vendors or + standardization organizations that are also treated like vendors + in the IANA-managed "SMI Network Management Private Enterprise + Codes" space wishing to implement a vendor-specific Diameter AVP + MUST use their own Vendor-ID along with their privately managed + AVP address space, guaranteeing that they will not collide with + any other vendor's vendor-specific AVP(s) or with future IETF + AVPs. + + + + + + +Fajardo, et al. Standards Track [Page 42] + +RFC 6733 Diameter Base Protocol October 2012 + + + A Vendor-ID value of zero (0) corresponds to the IETF-adopted AVP + values, as managed by IANA. Since the absence of the Vendor-ID + field implies that the AVP in question is not vendor specific, + implementations MUST NOT use the value of zero (0) for the + Vendor-ID field. + +4.2. Basic AVP Data Formats + + The Data field is zero or more octets and contains information + specific to the Attribute. The format and length of the Data field + is determined by the AVP Code and AVP Length fields. The format of + the Data field MUST be one of the following base data types or a data + type derived from the base data types. In the event that a new Basic + AVP Data Format is needed, a new version of this RFC MUST be created. + + OctetString + + The data contains arbitrary data of variable length. Unless + otherwise noted, the AVP Length field MUST be set to at least 8 + (12 if the 'V' bit is enabled). AVP values of this type that are + not a multiple of 4 octets in length are followed by the necessary + padding so that the next AVP (if any) will start on a 32-bit + boundary. + + Integer32 + + 32-bit signed value, in network byte order. The AVP Length field + MUST be set to 12 (16 if the 'V' bit is enabled). + + Integer64 + + 64-bit signed value, in network byte order. The AVP Length field + MUST be set to 16 (20 if the 'V' bit is enabled). + + Unsigned32 + + 32-bit unsigned value, in network byte order. The AVP Length + field MUST be set to 12 (16 if the 'V' bit is enabled). + + Unsigned64 + + 64-bit unsigned value, in network byte order. The AVP Length + field MUST be set to 16 (20 if the 'V' bit is enabled). + + + + + + + + +Fajardo, et al. Standards Track [Page 43] + +RFC 6733 Diameter Base Protocol October 2012 + + + Float32 + + This represents floating point values of single precision as + described by [FLOATPOINT]. The 32-bit value is transmitted in + network byte order. The AVP Length field MUST be set to 12 (16 if + the 'V' bit is enabled). + + Float64 + + This represents floating point values of double precision as + described by [FLOATPOINT]. The 64-bit value is transmitted in + network byte order. The AVP Length field MUST be set to 16 (20 if + the 'V' bit is enabled). + + Grouped + + The Data field is specified as a sequence of AVPs. These AVPs are + concatenated -- including their headers and padding -- in the + order in which they are specified and the result encapsulated in + the Data field. The AVP Length field is set to 8 (12 if the 'V' + bit is enabled) plus the total length of all included AVPs, + including their headers and padding. Thus, the AVP Length field + of an AVP of type Grouped is always a multiple of 4. + +4.3. Derived AVP Data Formats + + In addition to using the Basic AVP Data Formats, applications may + define data formats derived from the Basic AVP Data Formats. An + application that defines new Derived AVP Data Formats MUST include + them in a section titled "Derived AVP Data Formats", using the same + format as the definitions below. Each new definition MUST be either + defined or listed with a reference to the RFC that defines the + format. + +4.3.1. Common Derived AVP Data Formats + + The following are commonly used Derived AVP Data Formats. + + Address + + The Address format is derived from the OctetString Basic AVP + Format. It is a discriminated union representing, for example, a + 32-bit (IPv4) [RFC0791] or 128-bit (IPv6) [RFC4291] address, most + significant octet first. The first two octets of the Address AVP + represent the AddressType, which contains an Address Family, + defined in [IANAADFAM]. The AddressType is used to discriminate + the content and format of the remaining octets. + + + + +Fajardo, et al. Standards Track [Page 44] + +RFC 6733 Diameter Base Protocol October 2012 + + + Time + + The Time format is derived from the OctetString Basic AVP Format. + The string MUST contain four octets, in the same format as the + first four bytes are in the NTP timestamp format. The NTP + timestamp format is defined in Section 3 of [RFC5905]. + + This represents the number of seconds since 0h on 1 January 1900 + with respect to the Coordinated Universal Time (UTC). + + On 6h 28m 16s UTC, 7 February 2036, the time value will overflow. + Simple Network Time Protocol (SNTP) [RFC5905] describes a + procedure to extend the time to 2104. This procedure MUST be + supported by all Diameter nodes. + + UTF8String + + The UTF8String format is derived from the OctetString Basic AVP + Format. This is a human-readable string represented using the + ISO/IEC IS 10646-1 character set, encoded as an OctetString using + the UTF-8 transformation format [RFC3629]. + + Since additional code points are added by amendments to the 10646 + standard from time to time, implementations MUST be prepared to + encounter any code point from 0x00000001 to 0x7fffffff. Byte + sequences that do not correspond to the valid encoding of a code + point into UTF-8 charset or are outside this range are prohibited. + + The use of control codes SHOULD be avoided. When it is necessary + to represent a new line, the control code sequence CR LF SHOULD be + used. + + The use of leading or trailing white space SHOULD be avoided. + + For code points not directly supported by user interface hardware + or software, an alternative means of entry and display, such as + hexadecimal, MAY be provided. + + For information encoded in 7-bit US-ASCII, the UTF-8 charset is + identical to the US-ASCII charset. + + UTF-8 may require multiple bytes to represent a single character / + code point; thus, the length of a UTF8String in octets may be + different from the number of characters encoded. + + Note that the AVP Length field of an UTF8String is measured in + octets not characters. + + + + +Fajardo, et al. Standards Track [Page 45] + +RFC 6733 Diameter Base Protocol October 2012 + + + DiameterIdentity + + The DiameterIdentity format is derived from the OctetString Basic + AVP Format. + + DiameterIdentity = FQDN/Realm + + The DiameterIdentity value is used to uniquely identify either: + + * A Diameter node for purposes of duplicate connection and + routing loop detection. + + * A Realm to determine whether messages can be satisfied locally + or whether they must be routed or redirected. + + When a DiameterIdentity value is used to identify a Diameter node, + the contents of the string MUST be the Fully Qualified Domain Name + (FQDN) of the Diameter node. If multiple Diameter nodes run on + the same host, each Diameter node MUST be assigned a unique + DiameterIdentity. If a Diameter node can be identified by several + FQDNs, a single FQDN should be picked at startup and used as the + only DiameterIdentity for that node, whatever the connection on + which it is sent. In this document, note that DiameterIdentity is + in ASCII form in order to be compatible with existing DNS + infrastructure. See Appendix D for interactions between the + Diameter protocol and Internationalized Domain Names (IDNs). + + DiameterURI + + The DiameterURI MUST follow the Uniform Resource Identifiers (RFC + 3986) syntax [RFC3986] rules specified below: + + "aaa://" FQDN [ port ] [ transport ] [ protocol ] + + ; No transport security + + "aaas://" FQDN [ port ] [ transport ] [ protocol ] + + ; Transport security used + + FQDN = < Fully Qualified Domain Name > + + + + + + + + + + +Fajardo, et al. Standards Track [Page 46] + +RFC 6733 Diameter Base Protocol October 2012 + + + port = ":" 1*DIGIT + + ; One of the ports used to listen for + ; incoming connections. + ; If absent, the default Diameter port + ; (3868) is assumed if no transport + ; security is used and port 5658 when + ; transport security (TLS/TCP and DTLS/SCTP) + ; is used. + + transport = ";transport=" transport-protocol + + ; One of the transports used to listen + ; for incoming connections. If absent, + ; the default protocol is assumed to be TCP. + ; UDP MUST NOT be used when the aaa-protocol + ; field is set to diameter. + + transport-protocol = ( "tcp" / "sctp" / "udp" ) + + protocol = ";protocol=" aaa-protocol + + ; If absent, the default AAA protocol + ; is Diameter. + + aaa-protocol = ( "diameter" / "radius" / "tacacs+" ) + + The following are examples of valid Diameter host identities: + + aaa://host.example.com;transport=tcp + aaa://host.example.com:6666;transport=tcp + aaa://host.example.com;protocol=diameter + aaa://host.example.com:6666;protocol=diameter + aaa://host.example.com:6666;transport=tcp;protocol=diameter + aaa://host.example.com:1813;transport=udp;protocol=radius + + Enumerated + + The Enumerated format is derived from the Integer32 Basic AVP + Format. The definition contains a list of valid values and their + interpretation and is described in the Diameter application + introducing the AVP. + + + + + + + + + +Fajardo, et al. Standards Track [Page 47] + +RFC 6733 Diameter Base Protocol October 2012 + + + IPFilterRule + + The IPFilterRule format is derived from the OctetString Basic AVP + Format and uses the ASCII charset. The rule syntax is a modified + subset of ipfw(8) from FreeBSD. Packets may be filtered based on + the following information that is associated with it: + + Direction (in or out) + Source and destination IP address (possibly masked) + Protocol + Source and destination port (lists or ranges) + TCP flags + IP fragment flag + IP options + ICMP types + + Rules for the appropriate direction are evaluated in order, with the + first matched rule terminating the evaluation. Each packet is + evaluated once. If no rule matches, the packet is dropped if the + last rule evaluated was a permit, and passed if the last rule was a + deny. + + IPFilterRule filters MUST follow the format: + + action dir proto from src to dst [options] + + action permit - Allow packets that match the rule. + deny - Drop packets that match the rule. + + dir "in" is from the terminal, "out" is to the + terminal. + + proto An IP protocol specified by number. The "ip" + keyword means any protocol will match. + + src and dst <address/mask> [ports] + + The <address/mask> may be specified as: + ipno An IPv4 or IPv6 number in dotted- + quad or canonical IPv6 form. Only + this exact IP number will match the + rule. + + + + + + + + + +Fajardo, et al. Standards Track [Page 48] + +RFC 6733 Diameter Base Protocol October 2012 + + + ipno/bits An IP number as above with a mask + width of the form 192.0.2.10/24. In + this case, all IP numbers from + 192.0.2.0 to 192.0.2.255 will match. + The bit width MUST be valid for the + IP version, and the IP number MUST + NOT have bits set beyond the mask. + For a match to occur, the same IP + version must be present in the + packet that was used in describing + the IP address. To test for a + particular IP version, the bits part + can be set to zero. The keyword + "any" is 0.0.0.0/0 or the IPv6 + equivalent. The keyword "assigned" + is the address or set of addresses + assigned to the terminal. For IPv4, + a typical first rule is often "deny + in ip! assigned". + + The sense of the match can be inverted by + preceding an address with the not modifier (!), + causing all other addresses to be matched + instead. This does not affect the selection of + port numbers. + + With the TCP, UDP, and SCTP protocols, optional + ports may be specified as: + + {port/port-port}[,ports[,...]] + + The '-' notation specifies a range of ports + (including boundaries). + + Fragmented packets that have a non-zero offset + (i.e., not the first fragment) will never match + a rule that has one or more port + specifications. See the frag option for + details on matching fragmented packets. + + options: + frag Match if the packet is a fragment and this is not + the first fragment of the datagram. frag may not + be used in conjunction with either tcpflags or + TCP/UDP port specifications. + + + + + + +Fajardo, et al. Standards Track [Page 49] + +RFC 6733 Diameter Base Protocol October 2012 + + + ipoptions spec + Match if the IP header contains the comma-separated + list of options specified in spec. The + supported IP options are: + + ssrr (strict source route), lsrr (loose source + route), rr (record packet route), and ts + (timestamp). The absence of a particular option + may be denoted with a '!'. + + tcpoptions spec + Match if the TCP header contains the comma-separated + list of options specified in spec. The + supported TCP options are: + + mss (maximum segment size), window (tcp window + advertisement), sack (selective ack), ts (rfc1323 + timestamp), and cc (rfc1644 t/tcp connection + count). The absence of a particular option may + be denoted with a '!'. + + established + TCP packets only. Match packets that have the RST + or ACK bits set. + + setup TCP packets only. Match packets that have the SYN + bit set but no ACK bit. + + + tcpflags spec + TCP packets only. Match if the TCP header + contains the comma-separated list of flags + specified in spec. The supported TCP flags are: + + fin, syn, rst, psh, ack, and urg. The absence of a + particular flag may be denoted with a '!'. A rule + that contains a tcpflags specification can never + match a fragmented packet that has a non-zero + offset. See the frag option for details on + matching fragmented packets. + + icmptypes types + ICMP packets only. Match if the ICMP type is in + the list types. The list may be specified as any + combination of ranges or individual types + separated by commas. Both the numeric values and + the symbolic values listed below can be used. The + supported ICMP types are: + + + +Fajardo, et al. Standards Track [Page 50] + +RFC 6733 Diameter Base Protocol October 2012 + + + echo reply (0), destination unreachable (3), + source quench (4), redirect (5), echo request + (8), router advertisement (9), router + solicitation (10), time-to-live exceeded (11), IP + header bad (12), timestamp request (13), + timestamp reply (14), information request (15), + information reply (16), address mask request (17), + and address mask reply (18). + + There is one kind of packet that the access device MUST always + discard, that is an IP fragment with a fragment offset of one. This + is a valid packet, but it only has one use, to try to circumvent + firewalls. + + An access device that is unable to interpret or apply a deny rule + MUST terminate the session. An access device that is unable to + interpret or apply a permit rule MAY apply a more restrictive rule. + An access device MAY apply deny rules of its own before the supplied + rules, for example to protect the access device owner's + infrastructure. + +4.4. Grouped AVP Values + + The Diameter protocol allows AVP values of type 'Grouped'. This + implies that the Data field is actually a sequence of AVPs. It is + possible to include an AVP with a Grouped type within a Grouped type, + that is, to nest them. AVPs within an AVP of type Grouped have the + same padding requirements as non-Grouped AVPs, as defined in + Section 4.4. + + The AVP Code numbering space of all AVPs included in a Grouped AVP is + the same as for non-Grouped AVPs. Receivers of a Grouped AVP that + does not have the 'M' (mandatory) bit set and one or more of the + encapsulated AVPs within the group has the 'M' (mandatory) bit set + MAY simply be ignored if the Grouped AVP itself is unrecognized. The + rule applies even if the encapsulated AVP with its 'M' (mandatory) + bit set is further encapsulated within other sub-groups, i.e., other + Grouped AVPs embedded within the Grouped AVP. + + Every Grouped AVP definition MUST include a corresponding grammar, + using ABNF [RFC5234] (with modifications), as defined below. + + grouped-avp-def = "<" name ">" "::=" avp + + name-fmt = ALPHA *(ALPHA / DIGIT / "-") + + + + + + +Fajardo, et al. Standards Track [Page 51] + +RFC 6733 Diameter Base Protocol October 2012 + + + name = name-fmt + ; The name has to be the name of an AVP, + ; defined in the base or extended Diameter + ; specifications. + + avp = header *fixed *required *optional + + header = "<" "AVP-Header:" avpcode [vendor] ">" + + avpcode = 1*DIGIT + ; The AVP Code assigned to the Grouped AVP. + + vendor = 1*DIGIT + ; The Vendor-ID assigned to the Grouped AVP. + ; If absent, the default value of zero is + ; used. + +4.4.1. Example AVP with a Grouped Data Type + + The Example-AVP (AVP Code 999999) is of type Grouped and is used to + clarify how Grouped AVP values work. The Grouped Data field has the + following CCF grammar: + + Example-AVP ::= < AVP Header: 999999 > + { Origin-Host } + 1*{ Session-Id } + *[ AVP ] + + An Example-AVP with Grouped Data follows. + + The Origin-Host AVP (Section 6.3) is required. In this case: + + Origin-Host = "example.com". + + One or more Session-Ids must follow. Here there are two: + + Session-Id = + "grump.example.com:33041;23432;893;0AF3B81" + + Session-Id = + "grump.example.com:33054;23561;2358;0AF3B82" + + + + + + + + + + +Fajardo, et al. Standards Track [Page 52] + +RFC 6733 Diameter Base Protocol October 2012 + + + optional AVPs included are + + Recovery-Policy = <binary> + 2163bc1d0ad82371f6bc09484133c3f09ad74a0dd5346d54195a7cf0b35 + 2cabc881839a4fdcfbc1769e2677a4c1fb499284c5f70b48f58503a45c5 + c2d6943f82d5930f2b7c1da640f476f0e9c9572a50db8ea6e51e1c2c7bd + f8bb43dc995144b8dbe297ac739493946803e1cee3e15d9b765008a1b2a + cf4ac777c80041d72c01e691cf751dbf86e85f509f3988e5875dc905119 + 26841f00f0e29a6d1ddc1a842289d440268681e052b30fb638045f7779c + 1d873c784f054f688f5001559ecff64865ef975f3e60d2fd7966b8c7f92 + + Futuristic-Acct-Record = <binary> + fe19da5802acd98b07a5b86cb4d5d03f0314ab9ef1ad0b67111ff3b90a0 + 57fe29620bf3585fd2dd9fcc38ce62f6cc208c6163c008f4258d1bc88b8 + 17694a74ccad3ec69269461b14b2e7a4c111fb239e33714da207983f58c + 41d018d56fe938f3cbf089aac12a912a2f0d1923a9390e5f789cb2e5067 + d3427475e49968f841 + + The data for the optional AVPs is represented in hexadecimal form + since the format of these AVPs is not known at the time of definition + of the Example-AVP group nor (likely) at the time when the example + instance of this AVP is interpreted -- except by Diameter + implementations that support the same set of AVPs. The encoding + example illustrates how padding is used and how length fields are + calculated. Also, note that AVPs may be present in the Grouped AVP + value that the receiver cannot interpret (here, the Recover-Policy + and Futuristic-Acct-Record AVPs). The length of the Example-AVP is + the sum of all the length of the member AVPs, including their + padding, plus the Example-AVP header size. + + + + + + + + + + + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 53] + +RFC 6733 Diameter Base Protocol October 2012 + + + This AVP would be encoded as follows: + + 0 1 2 3 4 5 6 7 + +-------+-------+-------+-------+-------+-------+-------+-------+ + 0 | Example AVP Header (AVP Code = 999999), Length = 496 | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 8 | Origin-Host AVP Header (AVP Code = 264), Length = 19 | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 16 | 'e' | 'x' | 'a' | 'm' | 'p' | 'l' | 'e' | '.' | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 24 | 'c' | 'o' | 'm' |Padding| Session-Id AVP Header | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 32 | (AVP Code = 263), Length = 49 | 'g' | 'r' | 'u' | 'm' | + +-------+-------+-------+-------+-------+-------+-------+-------+ + . . . + +-------+-------+-------+-------+-------+-------+-------+-------+ + 72 | 'F' | '3' | 'B' | '8' | '1' |Padding|Padding|Padding| + +-------+-------+-------+-------+-------+-------+-------+-------+ + 80 | Session-Id AVP Header (AVP Code = 263), Length = 50 | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 88 | 'g' | 'r' | 'u' | 'm' | 'p' | '.' | 'e' | 'x' | + +-------+-------+-------+-------+-------+-------+-------+-------+ + . . . + +-------+-------+-------+-------+-------+-------+-------+-------+ + 120| '5' | '8' | ';' | '0' | 'A' | 'F' | '3' | 'B' | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 128| '8' | '2' |Padding|Padding| Recovery-Policy Header (AVP | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 136| Code = 8341), Length = 223 | 0x21 | 0x63 | 0xbc | 0x1d | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 144| 0x0a | 0xd8 | 0x23 | 0x71 | 0xf6 | 0xbc | 0x09 | 0x48 | + +-------+-------+-------+-------+-------+-------+-------+-------+ + . . . + +-------+-------+-------+-------+-------+-------+-------+-------+ + 352| 0x8c | 0x7f | 0x92 |Padding| Futuristic-Acct-Record Header | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 328|(AVP Code = 15930),Length = 137| 0xfe | 0x19 | 0xda | 0x58 | + +-------+-------+-------+-------+-------+-------+-------+-------+ + 336| 0x02 | 0xac | 0xd9 | 0x8b | 0x07 | 0xa5 | 0xb8 | 0xc6 | + +-------+-------+-------+-------+-------+-------+-------+-------+ + . . . + +-------+-------+-------+-------+-------+-------+-------+-------+ + 488| 0xe4 | 0x99 | 0x68 | 0xf8 | 0x41 |Padding|Padding|Padding| + +-------+-------+-------+-------+-------+-------+-------+-------+ + + + + + + + +Fajardo, et al. Standards Track [Page 54] + +RFC 6733 Diameter Base Protocol October 2012 + + +4.5. Diameter Base Protocol AVPs + + The following table describes the Diameter AVPs defined in the base + protocol, their AVP Code values, types, and possible flag values. + + Due to space constraints, the short form DiamIdent is used to + represent DiameterIdentity. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 55] + +RFC 6733 Diameter Base Protocol October 2012 + + + +----------+ + | AVP Flag | + | rules | + |----+-----| + AVP Section | |MUST | + Attribute Name Code Defined Data Type |MUST| NOT | + -----------------------------------------|----+-----| + Acct- 85 9.8.2 Unsigned32 | M | V | + Interim-Interval | | | + Accounting- 483 9.8.7 Enumerated | M | V | + Realtime-Required | | | + Acct- 50 9.8.5 UTF8String | M | V | + Multi-Session-Id | | | + Accounting- 485 9.8.3 Unsigned32 | M | V | + Record-Number | | | + Accounting- 480 9.8.1 Enumerated | M | V | + Record-Type | | | + Acct- 44 9.8.4 OctetString| M | V | + Session-Id | | | + Accounting- 287 9.8.6 Unsigned64 | M | V | + Sub-Session-Id | | | + Acct- 259 6.9 Unsigned32 | M | V | + Application-Id | | | + Auth- 258 6.8 Unsigned32 | M | V | + Application-Id | | | + Auth-Request- 274 8.7 Enumerated | M | V | + Type | | | + Authorization- 291 8.9 Unsigned32 | M | V | + Lifetime | | | + Auth-Grace- 276 8.10 Unsigned32 | M | V | + Period | | | + Auth-Session- 277 8.11 Enumerated | M | V | + State | | | + Re-Auth-Request- 285 8.12 Enumerated | M | V | + Type | | | + Class 25 8.20 OctetString| M | V | + Destination-Host 293 6.5 DiamIdent | M | V | + Destination- 283 6.6 DiamIdent | M | V | + Realm | | | + Disconnect-Cause 273 5.4.3 Enumerated | M | V | + Error-Message 281 7.3 UTF8String | | V,M | + Error-Reporting- 294 7.4 DiamIdent | | V,M | + Host | | | + Event-Timestamp 55 8.21 Time | M | V | + Experimental- 297 7.6 Grouped | M | V | + Result | | | + -----------------------------------------|----+-----| + + + + +Fajardo, et al. Standards Track [Page 56] + +RFC 6733 Diameter Base Protocol October 2012 + + + +----------+ + | AVP Flag | + | rules | + |----+-----| + AVP Section | |MUST | + Attribute Name Code Defined Data Type |MUST| NOT | + -----------------------------------------|----+-----| + Experimental- 298 7.7 Unsigned32 | M | V | + Result-Code | | | + Failed-AVP 279 7.5 Grouped | M | V | + Firmware- 267 5.3.4 Unsigned32 | | V,M | + Revision | | | + Host-IP-Address 257 5.3.5 Address | M | V | + Inband-Security | M | V | + -Id 299 6.10 Unsigned32 | | | + Multi-Round- 272 8.19 Unsigned32 | M | V | + Time-Out | | | + Origin-Host 264 6.3 DiamIdent | M | V | + Origin-Realm 296 6.4 DiamIdent | M | V | + Origin-State-Id 278 8.16 Unsigned32 | M | V | + Product-Name 269 5.3.7 UTF8String | | V,M | + Proxy-Host 280 6.7.3 DiamIdent | M | V | + Proxy-Info 284 6.7.2 Grouped | M | V | + Proxy-State 33 6.7.4 OctetString| M | V | + Redirect-Host 292 6.12 DiamURI | M | V | + Redirect-Host- 261 6.13 Enumerated | M | V | + Usage | | | + Redirect-Max- 262 6.14 Unsigned32 | M | V | + Cache-Time | | | + Result-Code 268 7.1 Unsigned32 | M | V | + Route-Record 282 6.7.1 DiamIdent | M | V | + Session-Id 263 8.8 UTF8String | M | V | + Session-Timeout 27 8.13 Unsigned32 | M | V | + Session-Binding 270 8.17 Unsigned32 | M | V | + Session-Server- 271 8.18 Enumerated | M | V | + Failover | | | + Supported- 265 5.3.6 Unsigned32 | M | V | + Vendor-Id | | | + Termination- 295 8.15 Enumerated | M | V | + Cause | | | + User-Name 1 8.14 UTF8String | M | V | + Vendor-Id 266 5.3.3 Unsigned32 | M | V | + Vendor-Specific- 260 6.11 Grouped | M | V | + Application-Id | | | + -----------------------------------------|----+-----| + + + + + + +Fajardo, et al. Standards Track [Page 57] + +RFC 6733 Diameter Base Protocol October 2012 + + +5. Diameter Peers + + This section describes how Diameter nodes establish connections and + communicate with peers. + +5.1. Peer Connections + + Connections between diameter peers are established using their valid + DiameterIdentity. A Diameter node initiating a connection to a peer + MUST know the peer's DiameterIdentity. Methods for discovering a + Diameter peer can be found in Section 5.2. + + Although a Diameter node may have many possible peers with which it + is able to communicate, it may not be economical to have an + established connection to all of them. At a minimum, a Diameter node + SHOULD have an established connection with two peers per realm, known + as the primary and secondary peers. Of course, a node MAY have + additional connections, if it is deemed necessary. Typically, all + messages for a realm are sent to the primary peer but, in the event + that failover procedures are invoked, any pending requests are sent + to the secondary peer. However, implementations are free to load + balance requests between a set of peers. + + Note that a given peer MAY act as a primary for a given realm while + acting as a secondary for another realm. + + When a peer is deemed suspect, which could occur for various reasons, + including not receiving a DWA within an allotted time frame, no new + requests should be forwarded to the peer, but failover procedures are + invoked. When an active peer is moved to this mode, additional + connections SHOULD be established to ensure that the necessary number + of active connections exists. + + There are two ways that a peer is removed from the suspect peer list: + + 1. The peer is no longer reachable, causing the transport connection + to be shut down. The peer is moved to the closed state. + + 2. Three watchdog messages are exchanged with accepted round-trip + times, and the connection to the peer is considered stabilized. + + In the event the peer being removed is either the primary or + secondary, an alternate peer SHOULD replace the deleted peer and + assume the role of either primary or secondary. + + + + + + + +Fajardo, et al. Standards Track [Page 58] + +RFC 6733 Diameter Base Protocol October 2012 + + +5.2. Diameter Peer Discovery + + Allowing for dynamic Diameter agent discovery makes possible simpler + and more robust deployment of Diameter services. In order to promote + interoperable implementations of Diameter peer discovery, the + following mechanisms (manual configuration and DNS) are described. + These are based on existing IETF standards. Both mechanisms MUST be + supported by all Diameter implementations; either MAY be used. + + There are two cases where Diameter peer discovery may be performed. + The first is when a Diameter client needs to discover a first-hop + Diameter agent. The second case is when a Diameter agent needs to + discover another agent for further handling of a Diameter operation. + In both cases, the following 'search order' is recommended: + + 1. The Diameter implementation consults its list of statically + (manually) configured Diameter agent locations. These will be + used if they exist and respond. + + 2. The Diameter implementation performs a NAPTR query for a server + in a particular realm. The Diameter implementation has to know, + in advance, in which realm to look for a Diameter agent. This + could be deduced, for example, from the 'realm' in an NAI on + which a Diameter implementation needed to perform a Diameter + operation. + + The NAPTR usage in Diameter follows the S-NAPTR DDDS application + [RFC3958] in which the SERVICE field includes tags for the + desired application and supported application protocol. The + application service tag for a Diameter application is 'aaa' and + the supported application protocol tags are 'diameter.tcp', + 'diameter.sctp', 'diameter.dtls', or 'diameter.tls.tcp' + [RFC6408]. + + The client can follow the resolution process defined by the + S-NAPTR DDDS [RFC3958] application to find a matching SRV, A, or + AAAA record of a suitable peer. The domain suffixes in the NAPTR + replacement field SHOULD match the domain of the original query. + An example can be found in Appendix B. + + 3. If no NAPTR records are found, the requester directly queries for + one of the following SRV records: for Diameter over TCP, use + "_diameter._tcp.realm"; for Diameter over TLS, use + "_diameters._tcp.realm"; for Diameter over SCTP, use + "_diameter._sctp.realm"; for Diameter over DTLS, use + "_diameters._sctp.realm". If SRV records are found, then the + requester can perform address record query (A RR's and/or AAAA + + + + +Fajardo, et al. Standards Track [Page 59] + +RFC 6733 Diameter Base Protocol October 2012 + + + RR's) for the target hostname specified in the SRV records + following the rules given in [RFC2782]. If no SRV records are + found, the requester gives up. + + If the server is using a site certificate, the domain name in the + NAPTR query and the domain name in the replacement field MUST both be + valid based on the site certificate handed out by the server in the + TLS/TCP and DTLS/SCTP or Internet Key Exchange Protocol (IKE) + exchange. Similarly, the domain name in the SRV query and the domain + name in the target in the SRV record MUST both be valid based on the + same site certificate. Otherwise, an attacker could modify the DNS + records to contain replacement values in a different domain, and the + client could not validate whether this was the desired behavior or + the result of an attack. + + Also, the Diameter peer MUST check to make sure that the discovered + peers are authorized to act in its role. Authentication via IKE or + TLS/TCP and DTLS/SCTP, or validation of DNS RRs via DNSSEC is not + sufficient to conclude this. For example, a web server may have + obtained a valid TLS/TCP and DTLS/SCTP certificate, and secured RRs + may be included in the DNS, but this does not imply that it is + authorized to act as a Diameter server. + + Authorization can be achieved, for example, by the configuration of a + Diameter server Certification Authority (CA). The server CA issues a + certificate to the Diameter server, which includes an Object + Identifier (OID) to indicate the subject is a Diameter server in the + Extended Key Usage extension [RFC5280]. This certificate is then + used during TLS/TCP, DTLS/SCTP, or IKE security negotiation. + However, note that, at the time of writing, no Diameter server + Certification Authorities exist. + + A dynamically discovered peer causes an entry in the peer table (see + Section 2.6) to be created. Note that entries created via DNS MUST + expire (or be refreshed) within the DNS Time to Live (TTL). If a + peer is discovered outside of the local realm, a routing table entry + (see Section 2.7) for the peer's realm is created. The routing table + entry's expiration MUST match the peer's expiration value. + +5.3. Capabilities Exchange + + When two Diameter peers establish a transport connection, they MUST + exchange the Capabilities Exchange messages, as specified in the peer + state machine (see Section 5.6). This message allows the discovery + of a peer's identity and its capabilities (protocol version number, + the identifiers of supported Diameter applications, security + mechanisms, etc.). + + + + +Fajardo, et al. Standards Track [Page 60] + +RFC 6733 Diameter Base Protocol October 2012 + + + The receiver only issues commands to its peers that have advertised + support for the Diameter application that defines the command. A + Diameter node MUST cache the supported Application Ids in order to + ensure that unrecognized commands and/or AVPs are not unnecessarily + sent to a peer. + + A receiver of a Capabilities-Exchange-Request (CER) message that does + not have any applications in common with the sender MUST return a + Capabilities-Exchange-Answer (CEA) with the Result-Code AVP set to + DIAMETER_NO_COMMON_APPLICATION and SHOULD disconnect the transport + layer connection. Note that receiving a CER or CEA from a peer + advertising itself as a relay (see Section 2.4) MUST be interpreted + as having common applications with the peer. + + The receiver of the Capabilities-Exchange-Request (CER) MUST + determine common applications by computing the intersection of its + own set of supported Application Ids against all of the + Application-Id AVPs (Auth-Application-Id, Acct-Application-Id, and + Vendor-Specific-Application-Id) present in the CER. The value of the + Vendor-Id AVP in the Vendor-Specific-Application-Id MUST NOT be used + during computation. The sender of the Capabilities-Exchange-Answer + (CEA) SHOULD include all of its supported applications as a hint to + the receiver regarding all of its application capabilities. + + Diameter implementations SHOULD first attempt to establish a TLS/TCP + and DTLS/SCTP connection prior to the CER/CEA exchange. This + protects the capabilities information of both peers. To support + older Diameter implementations that do not fully conform to this + document, the transport security MAY still be negotiated via an + Inband-Security AVP. In this case, the receiver of a Capabilities- + Exchange-Request (CER) message that does not have any security + mechanisms in common with the sender MUST return a Capabilities- + Exchange-Answer (CEA) with the Result-Code AVP set to + DIAMETER_NO_COMMON_SECURITY and SHOULD disconnect the transport layer + connection. + + CERs received from unknown peers MAY be silently discarded, or a CEA + MAY be issued with the Result-Code AVP set to DIAMETER_UNKNOWN_PEER. + In both cases, the transport connection is closed. If the local + policy permits receiving CERs from unknown hosts, a successful CEA + MAY be returned. If a CER from an unknown peer is answered with a + successful CEA, the lifetime of the peer entry is equal to the + lifetime of the transport connection. In case of a transport + failure, all the pending transactions destined to the unknown peer + can be discarded. + + The CER and CEA messages MUST NOT be proxied, redirected, or relayed. + + + + +Fajardo, et al. Standards Track [Page 61] + +RFC 6733 Diameter Base Protocol October 2012 + + + Since the CER/CEA messages cannot be proxied, it is still possible + that an upstream agent will receive a message for which it has no + available peers to handle the application that corresponds to the + Command Code. In such instances, the 'E' bit is set in the answer + message (Section 7) with the Result-Code AVP set to + DIAMETER_UNABLE_TO_DELIVER to inform the downstream agent to take + action (e.g., re-routing request to an alternate peer). + + With the exception of the Capabilities-Exchange-Request message, a + message of type Request that includes the Auth-Application-Id or + Acct-Application-Id AVPs, or a message with an application-specific + Command Code MAY only be forwarded to a host that has explicitly + advertised support for the application (or has advertised the Relay + Application Id). + +5.3.1. Capabilities-Exchange-Request + + The Capabilities-Exchange-Request (CER), indicated by the Command + Code set to 257 and the Command Flags' 'R' bit set, is sent to + exchange local capabilities. Upon detection of a transport failure, + this message MUST NOT be sent to an alternate peer. + + When Diameter is run over SCTP [RFC4960] or DTLS/SCTP [RFC6083], + which allow for connections to span multiple interfaces and multiple + IP addresses, the Capabilities-Exchange-Request message MUST contain + one Host-IP-Address AVP for each potential IP address that MAY be + locally used when transmitting Diameter messages. + + Message Format + + <CER> ::= < Diameter Header: 257, REQ > + { Origin-Host } + { Origin-Realm } + 1* { Host-IP-Address } + { Vendor-Id } + { Product-Name } + [ Origin-State-Id ] + * [ Supported-Vendor-Id ] + * [ Auth-Application-Id ] + * [ Inband-Security-Id ] + * [ Acct-Application-Id ] + * [ Vendor-Specific-Application-Id ] + [ Firmware-Revision ] + * [ AVP ] + + + + + + + +Fajardo, et al. Standards Track [Page 62] + +RFC 6733 Diameter Base Protocol October 2012 + + +5.3.2. Capabilities-Exchange-Answer + + The Capabilities-Exchange-Answer (CEA), indicated by the Command Code + set to 257 and the Command Flags' 'R' bit cleared, is sent in + response to a CER message. + + When Diameter is run over SCTP [RFC4960] or DTLS/SCTP [RFC6083], + which allow connections to span multiple interfaces, hence, multiple + IP addresses, the Capabilities-Exchange-Answer message MUST contain + one Host-IP-Address AVP for each potential IP address that MAY be + locally used when transmitting Diameter messages. + + Message Format + + <CEA> ::= < Diameter Header: 257 > + { Result-Code } + { Origin-Host } + { Origin-Realm } + 1* { Host-IP-Address } + { Vendor-Id } + { Product-Name } + [ Origin-State-Id ] + [ Error-Message ] + [ Failed-AVP ] + * [ Supported-Vendor-Id ] + * [ Auth-Application-Id ] + * [ Inband-Security-Id ] + * [ Acct-Application-Id ] + * [ Vendor-Specific-Application-Id ] + [ Firmware-Revision ] + * [ AVP ] + +5.3.3. Vendor-Id AVP + + The Vendor-Id AVP (AVP Code 266) is of type Unsigned32 and contains + the IANA "SMI Network Management Private Enterprise Codes" + [ENTERPRISE] value assigned to the Diameter Software vendor. It is + envisioned that the combination of the Vendor-Id, Product-Name + (Section 5.3.7), and Firmware-Revision (Section 5.3.4) AVPs may + provide useful debugging information. + + A Vendor-Id value of zero in the CER or CEA message is reserved and + indicates that this field is ignored. + + + + + + + + +Fajardo, et al. Standards Track [Page 63] + +RFC 6733 Diameter Base Protocol October 2012 + + +5.3.4. Firmware-Revision AVP + + The Firmware-Revision AVP (AVP Code 267) is of type Unsigned32 and is + used to inform a Diameter peer of the firmware revision of the + issuing device. + + For devices that do not have a firmware revision (general-purpose + computers running Diameter software modules, for instance), the + revision of the Diameter software module may be reported instead. + +5.3.5. Host-IP-Address AVP + + The Host-IP-Address AVP (AVP Code 257) is of type Address and is used + to inform a Diameter peer of the sender's IP address. All source + addresses that a Diameter node expects to use with SCTP [RFC4960] or + DTLS/SCTP [RFC6083] MUST be advertised in the CER and CEA messages by + including a Host-IP-Address AVP for each address. + +5.3.6. Supported-Vendor-Id AVP + + The Supported-Vendor-Id AVP (AVP Code 265) is of type Unsigned32 and + contains the IANA "SMI Network Management Private Enterprise Codes" + [ENTERPRISE] value assigned to a vendor other than the device vendor + but including the application vendor. This is used in the CER and + CEA messages in order to inform the peer that the sender supports (a + subset of) the Vendor-Specific AVPs defined by the vendor identified + in this AVP. The value of this AVP MUST NOT be set to zero. + Multiple instances of this AVP containing the same value SHOULD NOT + be sent. + +5.3.7. Product-Name AVP + + The Product-Name AVP (AVP Code 269) is of type UTF8String and + contains the vendor-assigned name for the product. The Product-Name + AVP SHOULD remain constant across firmware revisions for the same + product. + +5.4. Disconnecting Peer Connections + + When a Diameter node disconnects one of its transport connections, + its peer cannot know the reason for the disconnect and will most + likely assume that a connectivity problem occurred or that the peer + has rebooted. In these cases, the peer may periodically attempt to + reconnect, as stated in Section 2.1. In the event that the + disconnect was a result of either a shortage of internal resources or + simply that the node in question has no intentions of forwarding any + Diameter messages to the peer in the foreseeable future, a periodic + + + + +Fajardo, et al. Standards Track [Page 64] + +RFC 6733 Diameter Base Protocol October 2012 + + + connection request would not be welcomed. The Disconnection-Reason + AVP contains the reason the Diameter node issued the Disconnect-Peer- + Request message. + + The Disconnect-Peer-Request message is used by a Diameter node to + inform its peer of its intent to disconnect the transport layer and + that the peer shouldn't reconnect unless it has a valid reason to do + so (e.g., message to be forwarded). Upon receipt of the message, the + Disconnect-Peer-Answer message is returned, which SHOULD contain an + error if messages have recently been forwarded, and are likely in + flight, which would otherwise cause a race condition. + + The receiver of the Disconnect-Peer-Answer message initiates the + transport disconnect. The sender of the Disconnect-Peer-Answer + message should be able to detect the transport closure and clean up + the connection. + +5.4.1. Disconnect-Peer-Request + + The Disconnect-Peer-Request (DPR), indicated by the Command Code set + to 282 and the Command Flags' 'R' bit set, is sent to a peer to + inform it of its intentions to shut down the transport connection. + Upon detection of a transport failure, this message MUST NOT be sent + to an alternate peer. + + Message Format + + <DPR> ::= < Diameter Header: 282, REQ > + { Origin-Host } + { Origin-Realm } + { Disconnect-Cause } + * [ AVP ] + +5.4.2. Disconnect-Peer-Answer + + The Disconnect-Peer-Answer (DPA), indicated by the Command Code set + to 282 and the Command Flags' 'R' bit cleared, is sent as a response + to the Disconnect-Peer-Request message. Upon receipt of this + message, the transport connection is shut down. + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 65] + +RFC 6733 Diameter Base Protocol October 2012 + + + Message Format + + <DPA> ::= < Diameter Header: 282 > + { Result-Code } + { Origin-Host } + { Origin-Realm } + [ Error-Message ] + [ Failed-AVP ] + * [ AVP ] + + +5.4.3. Disconnect-Cause AVP + + The Disconnect-Cause AVP (AVP Code 273) is of type Enumerated. A + Diameter node MUST include this AVP in the Disconnect-Peer-Request + message to inform the peer of the reason for its intention to shut + down the transport connection. The following values are supported: + + REBOOTING 0 + A scheduled reboot is imminent. A receiver of a DPR with + above result code MAY attempt reconnection. + + BUSY 1 + The peer's internal resources are constrained, and it has + determined that the transport connection needs to be closed. + A receiver of a DPR with above result code SHOULD NOT attempt + reconnection. + + DO_NOT_WANT_TO_TALK_TO_YOU 2 + The peer has determined that it does not see a need for the + transport connection to exist, since it does not expect any + messages to be exchanged in the near future. A receiver of a + DPR with above result code SHOULD NOT attempt reconnection. + +5.5. Transport Failure Detection + + Given the nature of the Diameter protocol, it is recommended that + transport failures be detected as soon as possible. Detecting such + failures will minimize the occurrence of messages sent to unavailable + agents, resulting in unnecessary delays, and will provide better + failover performance. The Device-Watchdog-Request and Device- + Watchdog-Answer messages, defined in this section, are used to pro- + actively detect transport failures. + + + + + + + + +Fajardo, et al. Standards Track [Page 66] + +RFC 6733 Diameter Base Protocol October 2012 + + +5.5.1. Device-Watchdog-Request + + The Device-Watchdog-Request (DWR), indicated by the Command Code set + to 280 and the Command Flags' 'R' bit set, is sent to a peer when no + traffic has been exchanged between two peers (see Section 5.5.3). + Upon detection of a transport failure, this message MUST NOT be sent + to an alternate peer. + + Message Format + + <DWR> ::= < Diameter Header: 280, REQ > + { Origin-Host } + { Origin-Realm } + [ Origin-State-Id ] + * [ AVP ] + +5.5.2. Device-Watchdog-Answer + + The Device-Watchdog-Answer (DWA), indicated by the Command Code set + to 280 and the Command Flags' 'R' bit cleared, is sent as a response + to the Device-Watchdog-Request message. + + Message Format + + <DWA> ::= < Diameter Header: 280 > + { Result-Code } + { Origin-Host } + { Origin-Realm } + [ Error-Message ] + [ Failed-AVP ] + [ Origin-State-Id ] + * [ AVP ] + +5.5.3. Transport Failure Algorithm + + The transport failure algorithm is defined in [RFC3539]. All + Diameter implementations MUST support the algorithm defined in that + specification in order to be compliant to the Diameter base protocol. + +5.5.4. Failover and Failback Procedures + + In the event that a transport failure is detected with a peer, it is + necessary for all pending request messages to be forwarded to an + alternate agent, if possible. This is commonly referred to as + "failover". + + + + + + +Fajardo, et al. Standards Track [Page 67] + +RFC 6733 Diameter Base Protocol October 2012 + + + In order for a Diameter node to perform failover procedures, it is + necessary for the node to maintain a pending message queue for a + given peer. When an answer message is received, the corresponding + request is removed from the queue. The Hop-by-Hop Identifier field + is used to match the answer with the queued request. + + When a transport failure is detected, if possible, all messages in + the queue are sent to an alternate agent with the T flag set. On + booting a Diameter client or agent, the T flag is also set on any + remaining records in non-volatile storage that are still waiting to + be transmitted. An example of a case where it is not possible to + forward the message to an alternate server is when the message has a + fixed destination, and the unavailable peer is the message's final + destination (see Destination-Host AVP). Such an error requires that + the agent return an answer message with the 'E' bit set and the + Result-Code AVP set to DIAMETER_UNABLE_TO_DELIVER. + + It is important to note that multiple identical requests or answers + MAY be received as a result of a failover. The End-to-End Identifier + field in the Diameter header along with the Origin-Host AVP MUST be + used to identify duplicate messages. + + As described in Section 2.1, a connection request should be + periodically attempted with the failed peer in order to re-establish + the transport connection. Once a connection has been successfully + established, messages can once again be forwarded to the peer. This + is commonly referred to as "failback". + +5.6. Peer State Machine + + This section contains a finite state machine that MUST be observed by + all Diameter implementations. Each Diameter node MUST follow the + state machine described below when communicating with each peer. + Multiple actions are separated by commas, and may continue on + succeeding lines, as space requires. Similarly, state and next state + may also span multiple lines, as space requires. + + This state machine is closely coupled with the state machine + described in [RFC3539], which is used to open, close, failover, + probe, and reopen transport connections. In particular, note that + [RFC3539] requires the use of watchdog messages to probe connections. + For Diameter, DWR and DWA messages are to be used. + + The I- prefix is used to represent the initiator (connecting) + connection, while the R- prefix is used to represent the responder + (listening) connection. The lack of a prefix indicates that the + event or action is the same regardless of the connection on which the + event occurred. + + + +Fajardo, et al. Standards Track [Page 68] + +RFC 6733 Diameter Base Protocol October 2012 + + + The stable states that a state machine may be in are Closed, I-Open, + and R-Open; all other states are intermediate. Note that I-Open and + R-Open are equivalent except for whether the initiator or responder + transport connection is used for communication. + + A CER message is always sent on the initiating connection immediately + after the connection request is successfully completed. In the case + of an election, one of the two connections will shut down. The + responder connection will survive if the Origin-Host of the local + Diameter entity is higher than that of the peer; the initiator + connection will survive if the peer's Origin-Host is higher. All + subsequent messages are sent on the surviving connection. Note that + the results of an election on one peer are guaranteed to be the + inverse of the results on the other. + + For TLS/TCP and DTLS/SCTP usage, a TLS/TCP and DTLS/SCTP handshake + SHOULD begin when both ends are in the closed state prior to any + Diameter message exchanges. The TLS/TCP and DTLS/SCTP connection + SHOULD be established before sending any CER or CEA message to secure + and protect the capabilities information of both peers. The TLS/TCP + and DTLS/SCTP connection SHOULD be disconnected when the state + machine moves to the closed state. When connecting to responders + that do not conform to this document (i.e., older Diameter + implementations that are not prepared to received TLS/TCP and DTLS/ + SCTP connections in the closed state), the initial TLS/TCP and DTLS/ + SCTP connection attempt will fail. The initiator MAY then attempt to + connect via TCP or SCTP and initiate the TLS/TCP and DTLS/SCTP + handshake when both ends are in the open state. If the handshake is + successful, all further messages will be sent via TLS/TCP and DTLS/ + SCTP. If the handshake fails, both ends move to the closed state. + + The state machine constrains only the behavior of a Diameter + implementation as seen by Diameter peers through events on the wire. + + Any implementation that produces equivalent results is considered + compliant. + + + + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 69] + +RFC 6733 Diameter Base Protocol October 2012 + + + state event action next state + ----------------------------------------------------------------- + Closed Start I-Snd-Conn-Req Wait-Conn-Ack + R-Conn-CER R-Accept, R-Open + Process-CER, + R-Snd-CEA + + Wait-Conn-Ack I-Rcv-Conn-Ack I-Snd-CER Wait-I-CEA + I-Rcv-Conn-Nack Cleanup Closed + R-Conn-CER R-Accept, Wait-Conn-Ack/ + Process-CER Elect + Timeout Error Closed + + Wait-I-CEA I-Rcv-CEA Process-CEA I-Open + R-Conn-CER R-Accept, Wait-Returns + Process-CER, + Elect + I-Peer-Disc I-Disc Closed + I-Rcv-Non-CEA Error Closed + Timeout Error Closed + + Wait-Conn-Ack/ I-Rcv-Conn-Ack I-Snd-CER,Elect Wait-Returns + Elect I-Rcv-Conn-Nack R-Snd-CEA R-Open + R-Peer-Disc R-Disc Wait-Conn-Ack + R-Conn-CER R-Reject Wait-Conn-Ack/ + Elect + Timeout Error Closed + + Wait-Returns Win-Election I-Disc,R-Snd-CEA R-Open + I-Peer-Disc I-Disc, R-Open + R-Snd-CEA + I-Rcv-CEA R-Disc I-Open + R-Peer-Disc R-Disc Wait-I-CEA + R-Conn-CER R-Reject Wait-Returns + Timeout Error Closed + + R-Open Send-Message R-Snd-Message R-Open + R-Rcv-Message Process R-Open + R-Rcv-DWR Process-DWR, R-Open + R-Snd-DWA + R-Rcv-DWA Process-DWA R-Open + R-Conn-CER R-Reject R-Open + Stop R-Snd-DPR Closing + R-Rcv-DPR R-Snd-DPA Closing + R-Peer-Disc R-Disc Closed + + + + + + +Fajardo, et al. Standards Track [Page 70] + +RFC 6733 Diameter Base Protocol October 2012 + + + I-Open Send-Message I-Snd-Message I-Open + I-Rcv-Message Process I-Open + I-Rcv-DWR Process-DWR, I-Open + I-Snd-DWA + I-Rcv-DWA Process-DWA I-Open + R-Conn-CER R-Reject I-Open + Stop I-Snd-DPR Closing + I-Rcv-DPR I-Snd-DPA Closing + I-Peer-Disc I-Disc Closed + + Closing I-Rcv-DPA I-Disc Closed + R-Rcv-DPA R-Disc Closed + Timeout Error Closed + I-Peer-Disc I-Disc Closed + R-Peer-Disc R-Disc Closed + +5.6.1. Incoming Connections + + When a connection request is received from a Diameter peer, it is + not, in the general case, possible to know the identity of that peer + until a CER is received from it. This is because host and port + determine the identity of a Diameter peer; the source port of an + incoming connection is arbitrary. Upon receipt of a CER, the + identity of the connecting peer can be uniquely determined from the + Origin-Host. + + For this reason, a Diameter peer must employ logic separate from the + state machine to receive connection requests, accept them, and await + the CER. Once the CER arrives on a new connection, the Origin-Host + that identifies the peer is used to locate the state machine + associated with that peer, and the new connection and CER are passed + to the state machine as an R-Conn-CER event. + + The logic that handles incoming connections SHOULD close and discard + the connection if any message other than a CER arrives or if an + implementation-defined timeout occurs prior to receipt of CER. + + Because handling of incoming connections up to and including receipt + of a CER requires logic, separate from that of any individual state + machine associated with a particular peer, it is described separately + in this section rather than in the state machine above. + +5.6.2. Events + + Transitions and actions in the automaton are caused by events. In + this section, we will ignore the I- and R- prefixes, since the actual + event would be identical, but it would occur on one of two possible + connections. + + + +Fajardo, et al. Standards Track [Page 71] + +RFC 6733 Diameter Base Protocol October 2012 + + + Start The Diameter application has signaled that a + connection should be initiated with the peer. + + R-Conn-CER An acknowledgement is received stating that the + transport connection has been established, and the + associated CER has arrived. + + Rcv-Conn-Ack A positive acknowledgement is received confirming that + the transport connection is established. + + Rcv-Conn-Nack A negative acknowledgement was received stating that + the transport connection was not established. + + Timeout An application-defined timer has expired while waiting + for some event. + + Rcv-CER A CER message from the peer was received. + + Rcv-CEA A CEA message from the peer was received. + + Rcv-Non-CEA A message, other than a CEA, from the peer was + received. + + Peer-Disc A disconnection indication from the peer was received. + + Rcv-DPR A DPR message from the peer was received. + + Rcv-DPA A DPA message from the peer was received. + + Win-Election An election was held, and the local node was the + winner. + + Send-Message A message is to be sent. + + Rcv-Message A message other than CER, CEA, DPR, DPA, DWR, or DWA + was received. + + Stop The Diameter application has signaled that a + connection should be terminated (e.g., on system + shutdown). + +5.6.3. Actions + + Actions in the automaton are caused by events and typically indicate + the transmission of packets and/or an action to be taken on the + connection. In this section, we will ignore the I- and R- prefixes, + since the actual action would be identical, but it would occur on one + of two possible connections. + + + +Fajardo, et al. Standards Track [Page 72] + +RFC 6733 Diameter Base Protocol October 2012 + + + Snd-Conn-Req A transport connection is initiated with the peer. + + Accept The incoming connection associated with the R-Conn-CER + is accepted as the responder connection. + + Reject The incoming connection associated with the R-Conn-CER + is disconnected. + + Process-CER The CER associated with the R-Conn-CER is processed. + + Snd-CER A CER message is sent to the peer. + + Snd-CEA A CEA message is sent to the peer. + + Cleanup If necessary, the connection is shut down, and any + local resources are freed. + + Error The transport layer connection is disconnected, + either politely or abortively, in response to + an error condition. Local resources are freed. + + Process-CEA A received CEA is processed. + + Snd-DPR A DPR message is sent to the peer. + + Snd-DPA A DPA message is sent to the peer. + + Disc The transport layer connection is disconnected, + and local resources are freed. + + Elect An election occurs (see Section 5.6.4 for more + information). + + Snd-Message A message is sent. + + Snd-DWR A DWR message is sent. + + Snd-DWA A DWA message is sent. + + Process-DWR The DWR message is serviced. + + Process-DWA The DWA message is serviced. + + Process A message is serviced. + + + + + + + +Fajardo, et al. Standards Track [Page 73] + +RFC 6733 Diameter Base Protocol October 2012 + + +5.6.4. The Election Process + + The election is performed on the responder. The responder compares + the Origin-Host received in the CER with its own Origin-Host as two + streams of octets. If the local Origin-Host lexicographically + succeeds the received Origin-Host, a Win-Election event is issued + locally. Diameter identities are in ASCII form; therefore, the + lexical comparison is consistent with DNS case insensitivity, where + octets that fall in the ASCII range 'a' through 'z' MUST compare + equally to their uppercase counterparts between 'A' and 'Z'. See + Appendix D for interactions between the Diameter protocol and + Internationalized Domain Name (IDNs). + + The winner of the election MUST close the connection it initiated. + Historically, maintaining the responder side of a connection was more + efficient than maintaining the initiator side. However, current + practices makes this distinction irrelevant. + +6. Diameter Message Processing + + This section describes how Diameter requests and answers are created + and processed. + +6.1. Diameter Request Routing Overview + + A request is sent towards its final destination using one of the + following three combinations of the Destination-Realm and + Destination-Host AVPs: + + o A request that is not able to be proxied (such as a CER) MUST NOT + contain either Destination-Realm or Destination-Host AVPs. + + o A request that needs to be sent to a home server serving a + specific realm, but not to a specific server (such as the first + request of a series of round trips), MUST contain a Destination- + Realm AVP but MUST NOT contain a Destination-Host AVP. For + Diameter clients, the value of the Destination-Realm AVP MAY be + extracted from the User-Name AVP, or other methods. + + o Otherwise, a request that needs to be sent to a specific home + server among those serving a given realm MUST contain both the + Destination-Realm and Destination-Host AVPs. + + The Destination-Host AVP is used as described above when the + destination of the request is fixed, which includes: + + o Authentication requests that span multiple round trips. + + + + +Fajardo, et al. Standards Track [Page 74] + +RFC 6733 Diameter Base Protocol October 2012 + + + o A Diameter message that uses a security mechanism that makes use + of a pre-established session key shared between the source and the + final destination of the message. + + o Server-initiated messages that MUST be received by a specific + Diameter client (e.g., access device), such as the Abort-Session- + Request message, which is used to request that a particular user's + session be terminated. + + Note that an agent can only forward a request to a host described in + the Destination-Host AVP if the host in question is included in its + peer table (see Section 2.6). Otherwise, the request is routed based + on the Destination-Realm only (see Section 6.1.6). + + When a message is received, the message is processed in the following + order: + + o If the message is destined for the local host, the procedures + listed in Section 6.1.4 are followed. + + o If the message is intended for a Diameter peer with whom the local + host is able to directly communicate, the procedures listed in + Section 6.1.5 are followed. This is known as "Request + Forwarding". + + o The procedure listed in Section 6.1.6 is followed, which is known + as "Request Routing". + + o If none of the above are successful, an answer is returned with + the Result-Code set to DIAMETER_UNABLE_TO_DELIVER, with the 'E' + bit set. + + For routing of Diameter messages to work within an administrative + domain, all Diameter nodes within the realm MUST be peers. + + The overview contained in this section (6.1) is intended to provide + general guidelines to Diameter developers. Implementations are free + to use different methods than the ones described here as long as they + conform to the requirements specified in Sections 6.1.1 through + 6.1.9. See Section 7 for more details on error handling. + +6.1.1. Originating a Request + + When creating a request, in addition to any other procedures + described in the application definition for that specific request, + the following procedures MUST be followed: + + + + + +Fajardo, et al. Standards Track [Page 75] + +RFC 6733 Diameter Base Protocol October 2012 + + + o the Command Code is set to the appropriate value; + + o the 'R' bit is set; + + o the End-to-End Identifier is set to a locally unique value; + + o the Origin-Host and Origin-Realm AVPs MUST be set to the + appropriate values, used to identify the source of the message; + and + + o the Destination-Host and Destination-Realm AVPs MUST be set to the + appropriate values, as described in Section 6.1. + +6.1.2. Sending a Request + + When sending a request, originated either locally or as the result of + a forwarding or routing operation, the following procedures SHOULD be + followed: + + o The Hop-by-Hop Identifier SHOULD be set to a locally unique value. + + o The message SHOULD be saved in the list of pending requests. + + Other actions to perform on the message based on the particular role + the agent is playing are described in the following sections. + +6.1.3. Receiving Requests + + A relay or proxy agent MUST check for forwarding loops when receiving + requests. A loop is detected if the server finds its own identity in + a Route-Record AVP. When such an event occurs, the agent MUST answer + with the Result-Code AVP set to DIAMETER_LOOP_DETECTED. + +6.1.4. Processing Local Requests + + A request is known to be for local consumption when one of the + following conditions occurs: + + o The Destination-Host AVP contains the local host's identity; + + o The Destination-Host AVP is not present, the Destination-Realm AVP + contains a realm the server is configured to process locally, and + the Diameter application is locally supported; or + + o Both the Destination-Host and the Destination-Realm are not + present. + + + + + +Fajardo, et al. Standards Track [Page 76] + +RFC 6733 Diameter Base Protocol October 2012 + + + When a request is locally processed, the rules in Section 6.2 should + be used to generate the corresponding answer. + +6.1.5. Request Forwarding + + Request forwarding is done using the Diameter peer table. The + Diameter peer table contains all of the peers with which the local + node is able to directly communicate. + + When a request is received, and the host encoded in the Destination- + Host AVP is one that is present in the peer table, the message SHOULD + be forwarded to the peer. + +6.1.6. Request Routing + + Diameter request message routing is done via realms and Application + Ids. A Diameter message that may be forwarded by Diameter agents + (proxies, redirect agents, or relay agents) MUST include the target + realm in the Destination-Realm AVP. Request routing SHOULD rely on + the Destination-Realm AVP and the Application Id present in the + request message header to aid in the routing decision. The realm MAY + be retrieved from the User-Name AVP, which is in the form of a + Network Access Identifier (NAI). The realm portion of the NAI is + inserted in the Destination-Realm AVP. + + Diameter agents MAY have a list of locally supported realms and + applications, and they MAY have a list of externally supported realms + and applications. When a request is received that includes a realm + and/or application that is not locally supported, the message is + routed to the peer configured in the routing table (see Section 2.7). + + Realm names and Application Ids are the minimum supported routing + criteria, additional information may be needed to support redirect + semantics. + +6.1.7. Predictive Loop Avoidance + + Before forwarding or routing a request, Diameter agents, in addition + to performing the processing described in Section 6.1.3, SHOULD check + for the presence of a candidate route's peer identity in any of the + Route-Record AVPs. In the event of the agent detecting the presence + of a candidate route's peer identity in a Route-Record AVP, the agent + MUST ignore such a route for the Diameter request message and attempt + alternate routes if any exist. In case all the candidate routes are + eliminated by the above criteria, the agent SHOULD return a + DIAMETER_UNABLE_TO_DELIVER message. + + + + + +Fajardo, et al. Standards Track [Page 77] + +RFC 6733 Diameter Base Protocol October 2012 + + +6.1.8. Redirecting Requests + + When a redirect agent receives a request whose routing entry is set + to REDIRECT, it MUST reply with an answer message with the 'E' bit + set, while maintaining the Hop-by-Hop Identifier in the header, and + include the Result-Code AVP to DIAMETER_REDIRECT_INDICATION. Each of + the servers associated with the routing entry are added in a separate + Redirect-Host AVP. + + +------------------+ + | Diameter | + | Redirect Agent | + +------------------+ + ^ | 2. command + 'E' bit + 1. Request | | Result-Code = + [email protected] | | DIAMETER_REDIRECT_INDICATION + + | | Redirect-Host AVP(s) + | v + +-------------+ 3. Request +-------------+ + | example.com |------------->| example.net | + | Relay | | Diameter | + | Agent |<-------------| Server | + +-------------+ 4. Answer +-------------+ + + Figure 5: Diameter Redirect Agent + + The receiver of an answer message with the 'E' bit set and the + Result-Code AVP set to DIAMETER_REDIRECT_INDICATION uses the Hop-by- + Hop Identifier in the Diameter header to identify the request in the + pending message queue (see Section 5.5.4) that is to be redirected. + If no transport connection exists with the new peer, one is created, + and the request is sent directly to it. + + Multiple Redirect-Host AVPs are allowed. The receiver of the answer + message with the 'E' bit set selects exactly one of these hosts as + the destination of the redirected message. + + When the Redirect-Host-Usage AVP included in the answer message has a + non-zero value, a route entry for the redirect indications is created + and cached by the receiver. The redirect usage for such a route + entry is set by the value of Redirect-Host-Usage AVP and the lifetime + of the cached route entry is set by Redirect-Max-Cache-Time AVP + value. + + It is possible that multiple redirect indications can create multiple + cached route entries differing only in their redirect usage and the + peer to forward messages to. As an example, two(2) route entries + that are created by two(2) redirect indications results in two(2) + + + +Fajardo, et al. Standards Track [Page 78] + +RFC 6733 Diameter Base Protocol October 2012 + + + cached routes for the same realm and Application Id. However, one + has a redirect usage of ALL_SESSION, where matching requests will be + forwarded to one peer; the other has a redirect usage of ALL_REALM, + where request are forwarded to another peer. Therefore, an incoming + request that matches the realm and Application Id of both routes will + need additional resolution. In such a case, a routing precedence + rule MUST be used against the redirect usage value to resolve the + contention. The precedence rule can be found in Section 6.13. + +6.1.9. Relaying and Proxying Requests + + A relay or proxy agent MUST append a Route-Record AVP to all requests + forwarded. The AVP contains the identity of the peer from which the + request was received. + + The Hop-by-Hop Identifier in the request is saved and replaced with a + locally unique value. The source of the request is also saved, which + includes the IP address, port, and protocol. + + A relay or proxy agent MAY include the Proxy-Info AVP in requests if + it requires access to any local state information when the + corresponding response is received. The Proxy-Info AVP has security + implications as state information is distributed to other entities. + As such, it is RECOMMENDED that the content of the Proxy-Info AVP be + protected with cryptographic mechanisms, for example, by using a + keyed message digest such as HMAC-SHA1 [RFC2104]. Such a mechanism, + however, requires the management of keys, although only locally at + the Diameter server. Still, a full description of the management of + the keys used to protect the Proxy-Info AVP is beyond the scope of + this document. Below is a list of common recommendations: + + o The keys should be generated securely following the randomness + recommendations in [RFC4086]. + + o The keys and cryptographic protection algorithms should be at + least 128 bits in strength. + + o The keys should not be used for any other purpose than generating + and verifying instances of the Proxy-Info AVP. + + o The keys should be changed regularly. + + o The keys should be changed if the AVP format or cryptographic + protection algorithms change. + + The message is then forwarded to the next hop, as identified in the + routing table. + + + + +Fajardo, et al. Standards Track [Page 79] + +RFC 6733 Diameter Base Protocol October 2012 + + + Figure 6 provides an example of message routing using the procedures + listed in these sections. + + (Origin-Host=nas.example.net) (Origin-Host=nas.example.net) + (Origin-Realm=example.net) (Origin-Realm=example.net) + (Destination-Realm=example.com) (Destination-Realm=example.com) + (Route-Record=nas.example.net) + +------+ ------> +------+ ------> +------+ + | | (Request) | | (Request) | | + | NAS +-------------------+ DRL +-------------------+ HMS | + | | | | | | + +------+ <------ +------+ <------ +------+ + example.net (Answer) example.net (Answer) example.com + (Origin-Host=hms.example.com) (Origin-Host=hms.example.com) + (Origin-Realm=example.com) (Origin-Realm=example.com) + + Figure 6: Routing of Diameter messages + + Relay and proxy agents are not required to perform full inspection of + incoming messages. At a minimum, validation of the message header + and relevant routing AVPs has to be done when relaying messages. + Proxy agents may optionally perform more in-depth message validation + for applications in which it is interested. + +6.2. Diameter Answer Processing + + When a request is locally processed, the following procedures MUST be + applied to create the associated answer, in addition to any + additional procedures that MAY be discussed in the Diameter + application defining the command: + + o The same Hop-by-Hop Identifier in the request is used in the + answer. + + o The local host's identity is encoded in the Origin-Host AVP. + + o The Destination-Host and Destination-Realm AVPs MUST NOT be + present in the answer message. + + o The Result-Code AVP is added with its value indicating success or + failure. + + o If the Session-Id is present in the request, it MUST be included + in the answer. + + o Any Proxy-Info AVPs in the request MUST be added to the answer + message, in the same order they were present in the request. + + + + +Fajardo, et al. Standards Track [Page 80] + +RFC 6733 Diameter Base Protocol October 2012 + + + o The 'P' bit is set to the same value as the one in the request. + + o The same End-to-End identifier in the request is used in the + answer. + + Note that the error messages (see Section 7) are also subjected to + the above processing rules. + +6.2.1. Processing Received Answers + + A Diameter client or proxy MUST match the Hop-by-Hop Identifier in an + answer received against the list of pending requests. The + corresponding message should be removed from the list of pending + requests. It SHOULD ignore answers received that do not match a + known Hop-by-Hop Identifier. + +6.2.2. Relaying and Proxying Answers + + If the answer is for a request that was proxied or relayed, the agent + MUST restore the original value of the Diameter header's Hop-by-Hop + Identifier field. + + If the last Proxy-Info AVP in the message is targeted to the local + Diameter server, the AVP MUST be removed before the answer is + forwarded. + + If a relay or proxy agent receives an answer with a Result-Code AVP + indicating a failure, it MUST NOT modify the contents of the AVP. + Any additional local errors detected SHOULD be logged but not + reflected in the Result-Code AVP. If the agent receives an answer + message with a Result-Code AVP indicating success, and it wishes to + modify the AVP to indicate an error, it MUST modify the Result-Code + AVP to contain the appropriate error in the message destined towards + the access device as well as include the Error-Reporting-Host AVP; it + MUST also issue an STR on behalf of the access device towards the + Diameter server. + + The agent MUST then send the answer to the host that it received the + original request from. + +6.3. Origin-Host AVP + + The Origin-Host AVP (AVP Code 264) is of type DiameterIdentity, and + it MUST be present in all Diameter messages. This AVP identifies the + endpoint that originated the Diameter message. Relay agents MUST NOT + modify this AVP. + + + + + +Fajardo, et al. Standards Track [Page 81] + +RFC 6733 Diameter Base Protocol October 2012 + + + The value of the Origin-Host AVP is guaranteed to be unique within a + single host. + + Note that the Origin-Host AVP may resolve to more than one address as + the Diameter peer may support more than one address. + + This AVP SHOULD be placed as close to the Diameter header as + possible. + +6.4. Origin-Realm AVP + + The Origin-Realm AVP (AVP Code 296) is of type DiameterIdentity. + This AVP contains the Realm of the originator of any Diameter message + and MUST be present in all messages. + + This AVP SHOULD be placed as close to the Diameter header as + possible. + +6.5. Destination-Host AVP + + The Destination-Host AVP (AVP Code 293) is of type DiameterIdentity. + This AVP MUST be present in all unsolicited agent initiated messages, + MAY be present in request messages, and MUST NOT be present in answer + messages. + + The absence of the Destination-Host AVP will cause a message to be + sent to any Diameter server supporting the application within the + realm specified in Destination-Realm AVP. + + This AVP SHOULD be placed as close to the Diameter header as + possible. + +6.6. Destination-Realm AVP + + The Destination-Realm AVP (AVP Code 283) is of type DiameterIdentity + and contains the realm to which the message is to be routed. The + Destination-Realm AVP MUST NOT be present in answer messages. + Diameter clients insert the realm portion of the User-Name AVP. + Diameter servers initiating a request message use the value of the + Origin-Realm AVP from a previous message received from the intended + target host (unless it is known a priori). When present, the + Destination-Realm AVP is used to perform message routing decisions. + + The CCF for a request message that includes the Destination-Realm AVP + SHOULD list the Destination-Realm AVP as a required AVP (an AVP + indicated as {AVP}); otherwise, the message is inherently a non- + routable message. + + + + +Fajardo, et al. Standards Track [Page 82] + +RFC 6733 Diameter Base Protocol October 2012 + + + This AVP SHOULD be placed as close to the Diameter header as + possible. + +6.7. Routing AVPs + + The AVPs defined in this section are Diameter AVPs used for routing + purposes. These AVPs change as Diameter messages are processed by + agents. + +6.7.1. Route-Record AVP + + The Route-Record AVP (AVP Code 282) is of type DiameterIdentity. The + identity added in this AVP MUST be the same as the one received in + the Origin-Host of the Capabilities Exchange message. + +6.7.2. Proxy-Info AVP + + The Proxy-Info AVP (AVP Code 284) is of type Grouped. This AVP + contains the identity and local state information of the Diameter + node that creates and adds it to a message. The Grouped Data field + has the following CCF grammar: + + Proxy-Info ::= < AVP Header: 284 > + { Proxy-Host } + { Proxy-State } + * [ AVP ] + +6.7.3. Proxy-Host AVP + + The Proxy-Host AVP (AVP Code 280) is of type DiameterIdentity. This + AVP contains the identity of the host that added the Proxy-Info AVP. + +6.7.4. Proxy-State AVP + + The Proxy-State AVP (AVP Code 33) is of type OctetString. It + contains state information that would otherwise be stored at the + Diameter entity that created it. As such, this AVP MUST be treated + as opaque data by other Diameter entities. + +6.8. Auth-Application-Id AVP + + The Auth-Application-Id AVP (AVP Code 258) is of type Unsigned32 and + is used in order to advertise support of the Authentication and + Authorization portion of an application (see Section 2.4). If + present in a message other than CER and CEA, the value of the Auth- + Application-Id AVP MUST match the Application Id present in the + Diameter message header. + + + + +Fajardo, et al. Standards Track [Page 83] + +RFC 6733 Diameter Base Protocol October 2012 + + +6.9. Acct-Application-Id AVP + + The Acct-Application-Id AVP (AVP Code 259) is of type Unsigned32 and + is used in order to advertise support of the accounting portion of an + application (see Section 2.4). If present in a message other than + CER and CEA, the value of the Acct-Application-Id AVP MUST match the + Application Id present in the Diameter message header. + +6.10. Inband-Security-Id AVP + + The Inband-Security-Id AVP (AVP Code 299) is of type Unsigned32 and + is used in order to advertise support of the security portion of the + application. The use of this AVP in CER and CEA messages is NOT + RECOMMENDED. Instead, discovery of a Diameter entity's security + capabilities can be done either through static configuration or via + Diameter Peer Discovery as described in Section 5.2. + + The following values are supported: + + + NO_INBAND_SECURITY 0 + + This peer does not support TLS/TCP and DTLS/SCTP. This is the + default value, if the AVP is omitted. + + TLS 1 + + This node supports TLS/TCP [RFC5246] and DTLS/SCTP [RFC6083] + security. + +6.11. Vendor-Specific-Application-Id AVP + + The Vendor-Specific-Application-Id AVP (AVP Code 260) is of type + Grouped and is used to advertise support of a vendor-specific + Diameter application. Exactly one instance of either Auth- + Application-Id or Acct-Application-Id AVP MUST be present. The + Application Id carried by either Auth-Application-Id or Acct- + Application-Id AVP MUST comply with vendor-specific Application Id + assignment described in Section 11.3. It MUST also match the + Application Id present in the Diameter header except when used in a + CER or CEA message. + + The Vendor-Id AVP is an informational AVP pertaining to the vendor + who may have authorship of the vendor-specific Diameter application. + It MUST NOT be used as a means of defining a completely separate + vendor-specific Application Id space. + + + + + +Fajardo, et al. Standards Track [Page 84] + +RFC 6733 Diameter Base Protocol October 2012 + + + The Vendor-Specific-Application-Id AVP SHOULD be placed as close to + the Diameter header as possible. + + AVP Format + + <Vendor-Specific-Application-Id> ::= < AVP Header: 260 > + { Vendor-Id } + [ Auth-Application-Id ] + [ Acct-Application-Id ] + + A Vendor-Specific-Application-Id AVP MUST contain exactly one of + either Auth-Application-Id or Acct-Application-Id. If a Vendor- + Specific-Application-Id is received without one of these two AVPs, + then the recipient SHOULD issue an answer with a Result-Code set to + DIAMETER_MISSING_AVP. The answer SHOULD also include a Failed-AVP, + which MUST contain an example of an Auth-Application-Id AVP and an + Acct-Application-Id AVP. + + If a Vendor-Specific-Application-Id is received that contains both + Auth-Application-Id and Acct-Application-Id, then the recipient MUST + issue an answer with Result-Code set to + DIAMETER_AVP_OCCURS_TOO_MANY_TIMES. The answer MUST also include a + Failed-AVP, which MUST contain the received Auth-Application-Id AVP + and Acct-Application-Id AVP. + +6.12. Redirect-Host AVP + + The Redirect-Host AVP (AVP Code 292) is of type DiameterURI. One or + more instances of this AVP MUST be present if the answer message's + 'E' bit is set and the Result-Code AVP is set to + DIAMETER_REDIRECT_INDICATION. + + Upon receiving the above, the receiving Diameter node SHOULD forward + the request directly to one of the hosts identified in these AVPs. + The server contained in the selected Redirect-Host AVP SHOULD be used + for all messages matching the criteria set by the Redirect-Host-Usage + AVP. + +6.13. Redirect-Host-Usage AVP + + The Redirect-Host-Usage AVP (AVP Code 261) is of type Enumerated. + This AVP MAY be present in answer messages whose 'E' bit is set and + the Result-Code AVP is set to DIAMETER_REDIRECT_INDICATION. + + When present, this AVP provides hints about how the routing entry + resulting from the Redirect-Host is to be used. The following values + are supported: + + + + +Fajardo, et al. Standards Track [Page 85] + +RFC 6733 Diameter Base Protocol October 2012 + + + DONT_CACHE 0 + + The host specified in the Redirect-Host AVP SHOULD NOT be cached. + This is the default value. + + ALL_SESSION 1 + + All messages within the same session, as defined by the same value + of the Session-ID AVP SHOULD be sent to the host specified in the + Redirect-Host AVP. + + ALL_REALM 2 + + All messages destined for the realm requested SHOULD be sent to + the host specified in the Redirect-Host AVP. + + REALM_AND_APPLICATION 3 + + All messages for the application requested to the realm specified + SHOULD be sent to the host specified in the Redirect-Host AVP. + + ALL_APPLICATION 4 + + All messages for the application requested SHOULD be sent to the + host specified in the Redirect-Host AVP. + + ALL_HOST 5 + + All messages that would be sent to the host that generated the + Redirect-Host SHOULD be sent to the host specified in the + Redirect-Host AVP. + + ALL_USER 6 + + All messages for the user requested SHOULD be sent to the host + specified in the Redirect-Host AVP. + + When multiple cached routes are created by redirect indications and + they differ only in redirect usage and peers to forward requests to + (see Section 6.1.8), a precedence rule MUST be applied to the + redirect usage values of the cached routes during normal routing to + resolve contentions that may occur. The precedence rule is the order + that dictate which redirect usage should be considered before any + other as they appear. The order is as follows: + + + + + + + +Fajardo, et al. Standards Track [Page 86] + +RFC 6733 Diameter Base Protocol October 2012 + + + 1. ALL_SESSION + + 2. ALL_USER + + 3. REALM_AND_APPLICATION + + 4. ALL_REALM + + 5. ALL_APPLICATION + + 6. ALL_HOST + +6.14. Redirect-Max-Cache-Time AVP + + The Redirect-Max-Cache-Time AVP (AVP Code 262) is of type Unsigned32. + This AVP MUST be present in answer messages whose 'E' bit is set, + whose Result-Code AVP is set to DIAMETER_REDIRECT_INDICATION, and + whose Redirect-Host-Usage AVP set to a non-zero value. + + This AVP contains the maximum number of seconds the peer and route + table entries, created as a result of the Redirect-Host, SHOULD be + cached. Note that once a host is no longer reachable, any associated + cache, peer, and routing table entries MUST be deleted. + +7. Error Handling + + There are two different types of errors in Diameter; protocol errors + and application errors. A protocol error is one that occurs at the + base protocol level and MAY require per-hop attention (e.g., a + message routing error). Application errors, on the other hand, + generally occur due to a problem with a function specified in a + Diameter application (e.g., user authentication, missing AVP). + + Result-Code AVP values that are used to report protocol errors MUST + only be present in answer messages whose 'E' bit is set. When a + request message is received that causes a protocol error, an answer + message is returned with the 'E' bit set, and the Result-Code AVP is + set to the appropriate protocol error value. As the answer is sent + back towards the originator of the request, each proxy or relay agent + MAY take action on the message. + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 87] + +RFC 6733 Diameter Base Protocol October 2012 + + + 1. Request +---------+ Link Broken + +-------------------------->|Diameter |----///----+ + | +---------------------| | v + +------+--+ | 2. answer + 'E' set | Relay 2 | +--------+ + |Diameter |<-+ (Unable to Forward) +---------+ |Diameter| + | | | Home | + | Relay 1 |--+ +---------+ | Server | + +---------+ | 3. Request |Diameter | +--------+ + +-------------------->| | ^ + | Relay 3 |-----------+ + +---------+ + + Figure 7: Example of Protocol Error Causing Answer Message + + Figure 7 provides an example of a message forwarded upstream by a + Diameter relay. When the message is received by Relay 2, and it + detects that it cannot forward the request to the home server, an + answer message is returned with the 'E' bit set and the Result-Code + AVP set to DIAMETER_UNABLE_TO_DELIVER. Given that this error falls + within the protocol error category, Relay 1 would take special + action, and given the error, attempt to route the message through its + alternate Relay 3. + + +---------+ 1. Request +---------+ 2. Request +---------+ + | Access |------------>|Diameter |------------>|Diameter | + | | | | | Home | + | Device |<------------| Relay |<------------| Server | + +---------+ 4. Answer +---------+ 3. Answer +---------+ + (Missing AVP) (Missing AVP) + + Figure 8: Example of Application Error Answer Message + + Figure 8 provides an example of a Diameter message that caused an + application error. When application errors occur, the Diameter + entity reporting the error clears the 'R' bit in the Command Flags + and adds the Result-Code AVP with the proper value. Application + errors do not require any proxy or relay agent involvement; + therefore, the message would be forwarded back to the originator of + the request. + + In the case where the answer message itself contains errors, any + related session SHOULD be terminated by sending an STR or ASR + message. The Termination-Cause AVP in the STR MAY be filled with the + appropriate value to indicate the cause of the error. An application + MAY also send an application-specific request instead of an STR or + ASR message to signal the error in the case where no state is + maintained or to allow for some form of error recovery with the + corresponding Diameter entity. + + + +Fajardo, et al. Standards Track [Page 88] + +RFC 6733 Diameter Base Protocol October 2012 + + + There are certain Result-Code AVP application errors that require + additional AVPs to be present in the answer. In these cases, the + Diameter node that sets the Result-Code AVP to indicate the error + MUST add the AVPs. Examples are as follows: + + o A request with an unrecognized AVP is received with the 'M' bit + (Mandatory bit) set causes an answer to be sent with the Result- + Code AVP set to DIAMETER_AVP_UNSUPPORTED and the Failed-AVP AVP + containing the offending AVP. + + o A request with an AVP that is received with an unrecognized value + causes an answer to be returned with the Result-Code AVP set to + DIAMETER_INVALID_AVP_VALUE, with the Failed-AVP AVP containing the + AVP causing the error. + + o A received command that is missing AVPs that are defined as + required in the commands CCF; examples are AVPs indicated as + {AVP}. The receiver issues an answer with the Result-Code set to + DIAMETER_MISSING_AVP and creates an AVP with the AVP Code and + other fields set as expected in the missing AVP. The created AVP + is then added to the Failed-AVP AVP. + + The Result-Code AVP describes the error that the Diameter node + encountered in its processing. In case there are multiple errors, + the Diameter node MUST report only the first error it encountered + (detected possibly in some implementation-dependent order). The + specific errors that can be described by this AVP are described in + the following section. + +7.1. Result-Code AVP + + The Result-Code AVP (AVP Code 268) is of type Unsigned32 and + indicates whether a particular request was completed successfully or + an error occurred. All Diameter answer messages in IETF-defined + Diameter application specifications MUST include one Result-Code AVP. + A non-successful Result-Code AVP (one containing a non-2xxx value + other than DIAMETER_REDIRECT_INDICATION) MUST include the Error- + Reporting-Host AVP if the host setting the Result-Code AVP is + different from the identity encoded in the Origin-Host AVP. + + The Result-Code data field contains an IANA-managed 32-bit address + space representing errors (see Section 11.3.2). Diameter provides + the following classes of errors, all identified by the thousands + digit in the decimal notation: + + + + + + + +Fajardo, et al. Standards Track [Page 89] + +RFC 6733 Diameter Base Protocol October 2012 + + + o 1xxx (Informational) + + o 2xxx (Success) + + o 3xxx (Protocol Errors) + + o 4xxx (Transient Failures) + + o 5xxx (Permanent Failure) + + An unrecognized class (one whose first digit is not defined in this + section) MUST be handled as a permanent failure. + +7.1.1. Informational + + Errors that fall within this category are used to inform the + requester that a request could not be satisfied, and additional + action is required on its part before access is granted. + + DIAMETER_MULTI_ROUND_AUTH 1001 + + This informational error is returned by a Diameter server to + inform the access device that the authentication mechanism being + used requires multiple round trips, and a subsequent request needs + to be issued in order for access to be granted. + +7.1.2. Success + + Errors that fall within the Success category are used to inform a + peer that a request has been successfully completed. + + DIAMETER_SUCCESS 2001 + + The request was successfully completed. + + DIAMETER_LIMITED_SUCCESS 2002 + + When returned, the request was successfully completed, but + additional processing is required by the application in order to + provide service to the user. + +7.1.3. Protocol Errors + + Errors that fall within the Protocol Error category SHOULD be treated + on a per-hop basis, and Diameter proxies MAY attempt to correct the + error, if it is possible. Note that these errors MUST only be used + in answer messages whose 'E' bit is set. + + + + +Fajardo, et al. Standards Track [Page 90] + +RFC 6733 Diameter Base Protocol October 2012 + + + DIAMETER_COMMAND_UNSUPPORTED 3001 + + This error code is used when a Diameter entity receives a message + with a Command Code that it does not support. + + DIAMETER_UNABLE_TO_DELIVER 3002 + + This error is given when Diameter cannot deliver the message to + the destination, either because no host within the realm + supporting the required application was available to process the + request or because the Destination-Host AVP was given without the + associated Destination-Realm AVP. + + DIAMETER_REALM_NOT_SERVED 3003 + + The intended realm of the request is not recognized. + + DIAMETER_TOO_BUSY 3004 + + When returned, a Diameter node SHOULD attempt to send the message + to an alternate peer. This error MUST only be used when a + specific server is requested, and it cannot provide the requested + service. + + DIAMETER_LOOP_DETECTED 3005 + + An agent detected a loop while trying to get the message to the + intended recipient. The message MAY be sent to an alternate peer, + if one is available, but the peer reporting the error has + identified a configuration problem. + + DIAMETER_REDIRECT_INDICATION 3006 + + A redirect agent has determined that the request could not be + satisfied locally, and the initiator of the request SHOULD direct + the request directly to the server, whose contact information has + been added to the response. When set, the Redirect-Host AVP MUST + be present. + + DIAMETER_APPLICATION_UNSUPPORTED 3007 + + A request was sent for an application that is not supported. + + DIAMETER_INVALID_HDR_BITS 3008 + + A request was received whose bits in the Diameter header were set + either to an invalid combination or to a value that is + inconsistent with the Command Code's definition. + + + +Fajardo, et al. Standards Track [Page 91] + +RFC 6733 Diameter Base Protocol October 2012 + + + DIAMETER_INVALID_AVP_BITS 3009 + + A request was received that included an AVP whose flag bits are + set to an unrecognized value or that is inconsistent with the + AVP's definition. + + DIAMETER_UNKNOWN_PEER 3010 + + A CER was received from an unknown peer. + +7.1.4. Transient Failures + + Errors that fall within the transient failures category are used to + inform a peer that the request could not be satisfied at the time it + was received but MAY be able to satisfy the request in the future. + Note that these errors MUST be used in answer messages whose 'E' bit + is not set. + + DIAMETER_AUTHENTICATION_REJECTED 4001 + + The authentication process for the user failed, most likely due to + an invalid password used by the user. Further attempts MUST only + be tried after prompting the user for a new password. + + DIAMETER_OUT_OF_SPACE 4002 + + A Diameter node received the accounting request but was unable to + commit it to stable storage due to a temporary lack of space. + + ELECTION_LOST 4003 + + The peer has determined that it has lost the election process and + has therefore disconnected the transport connection. + +7.1.5. Permanent Failures + + Errors that fall within the permanent failures category are used to + inform the peer that the request failed and should not be attempted + again. Note that these errors SHOULD be used in answer messages + whose 'E' bit is not set. In error conditions where it is not + possible or efficient to compose application-specific answer grammar, + answer messages with the 'E' bit set and which comply to the grammar + described in Section 7.2 MAY also be used for permanent errors. + + + + + + + + +Fajardo, et al. Standards Track [Page 92] + +RFC 6733 Diameter Base Protocol October 2012 + + + DIAMETER_AVP_UNSUPPORTED 5001 + + The peer received a message that contained an AVP that is not + recognized or supported and was marked with the 'M' (Mandatory) + bit. A Diameter message with this error MUST contain one or more + Failed-AVP AVPs containing the AVPs that caused the failure. + + DIAMETER_UNKNOWN_SESSION_ID 5002 + + The request contained an unknown Session-Id. + + DIAMETER_AUTHORIZATION_REJECTED 5003 + + A request was received for which the user could not be authorized. + This error could occur if the service requested is not permitted + to the user. + + DIAMETER_INVALID_AVP_VALUE 5004 + + The request contained an AVP with an invalid value in its data + portion. A Diameter message indicating this error MUST include + the offending AVPs within a Failed-AVP AVP. + + DIAMETER_MISSING_AVP 5005 + + The request did not contain an AVP that is required by the Command + Code definition. If this value is sent in the Result-Code AVP, a + Failed-AVP AVP SHOULD be included in the message. The Failed-AVP + AVP MUST contain an example of the missing AVP complete with the + Vendor-Id if applicable. The value field of the missing AVP + should be of correct minimum length and contain zeroes. + + DIAMETER_RESOURCES_EXCEEDED 5006 + + A request was received that cannot be authorized because the user + has already expended allowed resources. An example of this error + condition is when a user that is restricted to one dial-up PPP + port attempts to establish a second PPP connection. + + DIAMETER_CONTRADICTING_AVPS 5007 + + The Home Diameter server has detected AVPs in the request that + contradicted each other, and it is not willing to provide service + to the user. The Failed-AVP AVP MUST be present, which contain + the AVPs that contradicted each other. + + + + + + +Fajardo, et al. Standards Track [Page 93] + +RFC 6733 Diameter Base Protocol October 2012 + + + DIAMETER_AVP_NOT_ALLOWED 5008 + + A message was received with an AVP that MUST NOT be present. The + Failed-AVP AVP MUST be included and contain a copy of the + offending AVP. + + DIAMETER_AVP_OCCURS_TOO_MANY_TIMES 5009 + + A message was received that included an AVP that appeared more + often than permitted in the message definition. The Failed-AVP + AVP MUST be included and contain a copy of the first instance of + the offending AVP that exceeded the maximum number of occurrences. + + DIAMETER_NO_COMMON_APPLICATION 5010 + + This error is returned by a Diameter node that receives a CER + whereby no applications are common between the CER sending peer + and the CER receiving peer. + + DIAMETER_UNSUPPORTED_VERSION 5011 + + This error is returned when a request was received, whose version + number is unsupported. + + DIAMETER_UNABLE_TO_COMPLY 5012 + + This error is returned when a request is rejected for unspecified + reasons. + + DIAMETER_INVALID_BIT_IN_HEADER 5013 + + This error is returned when a reserved bit in the Diameter header + is set to one (1) or the bits in the Diameter header are set + incorrectly. + + DIAMETER_INVALID_AVP_LENGTH 5014 + + The request contained an AVP with an invalid length. A Diameter + message indicating this error MUST include the offending AVPs + within a Failed-AVP AVP. In cases where the erroneous AVP length + value exceeds the message length or is less than the minimum AVP + header length, it is sufficient to include the offending AVP + header and a zero filled payload of the minimum required length + for the payloads data type. If the AVP is a Grouped AVP, the + Grouped AVP header with an empty payload would be sufficient to + indicate the offending AVP. In the case where the offending AVP + header cannot be fully decoded when the AVP length is less than + + + + +Fajardo, et al. Standards Track [Page 94] + +RFC 6733 Diameter Base Protocol October 2012 + + + the minimum AVP header length, it is sufficient to include an + offending AVP header that is formulated by padding the incomplete + AVP header with zero up to the minimum AVP header length. + + DIAMETER_INVALID_MESSAGE_LENGTH 5015 + + This error is returned when a request is received with an invalid + message length. + + DIAMETER_INVALID_AVP_BIT_COMBO 5016 + + The request contained an AVP with which is not allowed to have the + given value in the AVP Flags field. A Diameter message indicating + this error MUST include the offending AVPs within a Failed-AVP + AVP. + + DIAMETER_NO_COMMON_SECURITY 5017 + + This error is returned when a CER message is received, and there + are no common security mechanisms supported between the peers. A + Capabilities-Exchange-Answer (CEA) message MUST be returned with + the Result-Code AVP set to DIAMETER_NO_COMMON_SECURITY. + +7.2. Error Bit + + The 'E' (Error Bit) in the Diameter header is set when the request + caused a protocol-related error (see Section 7.1.3). A message with + the 'E' bit MUST NOT be sent as a response to an answer message. + Note that a message with the 'E' bit set is still subjected to the + processing rules defined in Section 6.2. When set, the answer + message will not conform to the CCF specification for the command; + instead, it and will conform to the following CCF: + + Message Format + + <answer-message> ::= < Diameter Header: code, ERR [, PXY] > + 0*1< Session-Id > + { Origin-Host } + { Origin-Realm } + { Result-Code } + [ Origin-State-Id ] + [ Error-Message ] + [ Error-Reporting-Host ] + [ Failed-AVP ] + [ Experimental-Result ] + * [ Proxy-Info ] + * [ AVP ] + + + + +Fajardo, et al. Standards Track [Page 95] + +RFC 6733 Diameter Base Protocol October 2012 + + + Note that the code used in the header is the same than the one found + in the request message, but with the 'R' bit cleared and the 'E' bit + set. The 'P' bit in the header is set to the same value as the one + found in the request message. + +7.3. Error-Message AVP + + The Error-Message AVP (AVP Code 281) is of type UTF8String. It MAY + accompany a Result-Code AVP as a human-readable error message. The + Error-Message AVP is not intended to be useful in an environment + where error messages are processed automatically. It SHOULD NOT be + expected that the content of this AVP be parsed by network entities. + +7.4. Error-Reporting-Host AVP + + The Error-Reporting-Host AVP (AVP Code 294) is of type + DiameterIdentity. This AVP contains the identity of the Diameter + host that sent the Result-Code AVP to a value other than 2001 + (Success), only if the host setting the Result-Code is different from + the one encoded in the Origin-Host AVP. This AVP is intended to be + used for troubleshooting purposes, and it MUST be set when the + Result-Code AVP indicates a failure. + +7.5. Failed-AVP AVP + + The Failed-AVP AVP (AVP Code 279) is of type Grouped and provides + debugging information in cases where a request is rejected or not + fully processed due to erroneous information in a specific AVP. The + value of the Result-Code AVP will provide information on the reason + for the Failed-AVP AVP. A Diameter answer message SHOULD contain an + instance of the Failed-AVP AVP that corresponds to the error + indicated by the Result-Code AVP. For practical purposes, this + Failed-AVP would typically refer to the first AVP processing error + that a Diameter node encounters. + + The possible reasons for this AVP are the presence of an improperly + constructed AVP, an unsupported or unrecognized AVP, an invalid AVP + value, the omission of a required AVP, the presence of an explicitly + excluded AVP (see tables in Section 10) or the presence of two or + more occurrences of an AVP that is restricted to 0, 1, or 0-1 + occurrences. + + A Diameter message SHOULD contain one Failed-AVP AVP, containing the + entire AVP that could not be processed successfully. If the failure + reason is omission of a required AVP, an AVP with the missing AVP + code, the missing Vendor-Id, and a zero-filled payload of the minimum + required length for the omitted AVP will be added. If the failure + reason is an invalid AVP length where the reported length is less + + + +Fajardo, et al. Standards Track [Page 96] + +RFC 6733 Diameter Base Protocol October 2012 + + + than the minimum AVP header length or greater than the reported + message length, a copy of the offending AVP header and a zero-filled + payload of the minimum required length SHOULD be added. + + In the case where the offending AVP is embedded within a Grouped AVP, + the Failed-AVP MAY contain the grouped AVP, which in turn contains + the single offending AVP. The same method MAY be employed if the + grouped AVP itself is embedded in yet another grouped AVP and so on. + In this case, the Failed-AVP MAY contain the grouped AVP hierarchy up + to the single offending AVP. This enables the recipient to detect + the location of the offending AVP when embedded in a group. + + AVP Format + + <Failed-AVP> ::= < AVP Header: 279 > + 1* {AVP} + +7.6. Experimental-Result AVP + + The Experimental-Result AVP (AVP Code 297) is of type Grouped, and + indicates whether a particular vendor-specific request was completed + successfully or whether an error occurred. This AVP has the + following structure: + + AVP Format + + Experimental-Result ::= < AVP Header: 297 > + { Vendor-Id } + { Experimental-Result-Code } + + The Vendor-Id AVP (see Section 5.3.3) in this grouped AVP identifies + the vendor responsible for the assignment of the result code that + follows. All Diameter answer messages defined in vendor-specific + applications MUST include either one Result-Code AVP or one + Experimental-Result AVP. + +7.7. Experimental-Result-Code AVP + + The Experimental-Result-Code AVP (AVP Code 298) is of type Unsigned32 + and contains a vendor-assigned value representing the result of + processing the request. + + It is recommended that vendor-specific result codes follow the same + conventions given for the Result-Code AVP regarding the different + types of result codes and the handling of errors (for non-2xxx + values). + + + + + +Fajardo, et al. Standards Track [Page 97] + +RFC 6733 Diameter Base Protocol October 2012 + + +8. Diameter User Sessions + + In general, Diameter can provide two different types of services to + applications. The first involves authentication and authorization, + and it can optionally make use of accounting. The second only makes + use of accounting. + + When a service makes use of the authentication and/or authorization + portion of an application, and a user requests access to the network, + the Diameter client issues an auth request to its local server. The + auth request is defined in a service-specific Diameter application + (e.g., NASREQ). The request contains a Session-Id AVP, which is used + in subsequent messages (e.g., subsequent authorization, accounting, + etc.) relating to the user's session. The Session-Id AVP is a means + for the client and servers to correlate a Diameter message with a + user session. + + When a Diameter server authorizes a user to implement network + resources for a finite amount of time, and it is willing to extend + the authorization via a future request, it MUST add the + Authorization- Lifetime AVP to the answer message. The + Authorization-Lifetime AVP defines the maximum number of seconds a + user MAY make use of the resources before another authorization + request is expected by the server. The Auth-Grace-Period AVP + contains the number of seconds following the expiration of the + Authorization-Lifetime, after which the server will release all state + information related to the user's session. Note that if payment for + services is expected by the serving realm from the user's home realm, + the Authorization-Lifetime AVP, combined with the Auth-Grace-Period + AVP, implies the maximum length of the session for which the home + realm is willing to be fiscally responsible. Services provided past + the expiration of the Authorization-Lifetime and Auth-Grace-Period + AVPs are the responsibility of the access device. Of course, the + actual cost of services rendered is clearly outside the scope of the + protocol. + + An access device that does not expect to send a re-authorization or a + session termination request to the server MAY include the Auth- + Session-State AVP with the value set to NO_STATE_MAINTAINED as a hint + to the server. If the server accepts the hint, it agrees that since + no session termination message will be received once service to the + user is terminated, it cannot maintain state for the session. If the + answer message from the server contains a different value in the + Auth-Session-State AVP (or the default value if the AVP is absent), + the access device MUST follow the server's directives. Note that the + value NO_STATE_MAINTAINED MUST NOT be set in subsequent re- + authorization requests and answers. + + + + +Fajardo, et al. Standards Track [Page 98] + +RFC 6733 Diameter Base Protocol October 2012 + + + The base protocol does not include any authorization request + messages, since these are largely application-specific and are + defined in a Diameter application document. However, the base + protocol does define a set of messages that are used to terminate + user sessions. These are used to allow servers that maintain state + information to free resources. + + When a service only makes use of the accounting portion of the + Diameter protocol, even in combination with an application, the + Session-Id is still used to identify user sessions. However, the + session termination messages are not used, since a session is + signaled as being terminated by issuing an accounting stop message. + + Diameter may also be used for services that cannot be easily + categorized as authentication, authorization, or accounting (e.g., + certain Third Generation Partnership Project Internet Multimedia + System (3GPP IMS) interfaces). In such cases, the finite state + machine defined in subsequent sections may not be applicable. + Therefore, the application itself MAY need to define its own finite + state machine. However, such application-specific state machines + SHOULD follow the general state machine framework outlined in this + document such as the use of Session-Id AVPs and the use of STR/STA, + ASR/ASA messages for stateful sessions. + +8.1. Authorization Session State Machine + + This section contains a set of finite state machines, which represent + the life cycle of Diameter sessions and which MUST be observed by all + Diameter implementations that make use of the authentication and/or + authorization portion of a Diameter application. The term "Service- + Specific" below refers to a message defined in a Diameter application + (e.g., Mobile IPv4, NASREQ). + + There are four different authorization session state machines + supported in the Diameter base protocol. The first two describe a + session in which the server is maintaining session state, indicated + by the value of the Auth-Session-State AVP (or its absence). One + describes the session from a client perspective, the other from a + server perspective. The second two state machines are used when the + server does not maintain session state. Here again, one describes + the session from a client perspective, the other from a server + perspective. + + When a session is moved to the Idle state, any resources that were + allocated for the particular session must be released. Any event not + listed in the state machines MUST be considered an error condition, + and an answer, if applicable, MUST be returned to the originator of + the message. + + + +Fajardo, et al. Standards Track [Page 99] + +RFC 6733 Diameter Base Protocol October 2012 + + + In the case that an application does not support re-auth, the state + transitions related to server-initiated re-auth, when both client and + server sessions maintain state (e.g., Send RAR, Pending, Receive + RAA), MAY be ignored. + + In the state table, the event "Failure to send X" means that the + Diameter agent is unable to send command X to the desired + destination. This could be due to the peer being down or due to the + peer sending back a transient failure or temporary protocol error + notification DIAMETER_TOO_BUSY or DIAMETER_LOOP_DETECTED in the + Result-Code AVP of the corresponding Answer command. The event 'X + successfully sent' is the complement of 'Failure to send X'. + + The following state machine is observed by a client when state is + maintained on the server: + + CLIENT, STATEFUL + State Event Action New State + --------------------------------------------------------------- + Idle Client or device requests Send Pending + access service- + specific + auth req + + Idle ASR Received Send ASA Idle + for unknown session with + Result-Code = + UNKNOWN_ + SESSION_ID + + Idle RAR Received Send RAA Idle + for unknown session with + Result-Code = + UNKNOWN_ + SESSION_ID + + Pending Successful service-specific Grant Open + authorization answer Access + received with default + Auth-Session-State value + + Pending Successful service-specific Sent STR Discon + authorization answer received, + but service not provided + + Pending Error processing successful Sent STR Discon + service-specific authorization + answer + + + +Fajardo, et al. Standards Track [Page 100] + +RFC 6733 Diameter Base Protocol October 2012 + + + Pending Failed service-specific Clean up Idle + authorization answer received + + Open User or client device Send Open + requests access to service service- + specific + auth req + + Open Successful service-specific Provide Open + authorization answer received service + + Open Failed service-specific Discon. Idle + authorization answer user/device + received. + + Open RAR received and client will Send RAA Open + perform subsequent re-auth with + Result-Code = + SUCCESS + + Open RAR received and client will Send RAA Idle + not perform subsequent with + re-auth Result-Code != + SUCCESS, + Discon. + user/device + + Open Session-Timeout expires on Send STR Discon + access device + + Open ASR received, Send ASA Discon + client will comply with + with request to end the Result-Code = + session = SUCCESS, + Send STR. + + Open ASR Received, Send ASA Open + client will not comply with + with request to end the Result-Code != + session != SUCCESS + + Open Authorization-Lifetime + Send STR Discon + Auth-Grace-Period expires on + access device + + Discon ASR received Send ASA Discon + + + + + +Fajardo, et al. Standards Track [Page 101] + +RFC 6733 Diameter Base Protocol October 2012 + + + Discon STA received Discon. Idle + user/device + + The following state machine is observed by a server when it is + maintaining state for the session: + + SERVER, STATEFUL + State Event Action New State + --------------------------------------------------------------- + Idle Service-specific authorization Send Open + request received, and successful + user is authorized service- + specific + answer + + Idle Service-specific authorization Send Idle + request received, and failed + user is not authorized service- + specific + answer + + Open Service-specific authorization Send Open + request received, and user successful + is authorized service- + specific + answer + + Open Service-specific authorization Send Idle + request received, and user failed + is not authorized service- + specific + answer, + Clean up + + Open Home server wants to confirm Send RAR Pending + authentication and/or + authorization of the user + + Pending Received RAA with a failed Clean up Idle + Result-Code + + Pending Received RAA with Result-Code Update Open + = SUCCESS session + + Open Home server wants to Send ASR Discon + terminate the service + + + + + +Fajardo, et al. Standards Track [Page 102] + +RFC 6733 Diameter Base Protocol October 2012 + + + Open Authorization-Lifetime (and Clean up Idle + Auth-Grace-Period) expires + on home server + + Open Session-Timeout expires on Clean up Idle + home server + + Discon Failure to send ASR Wait, Discon + resend ASR + + Discon ASR successfully sent and Clean up Idle + ASA Received with Result-Code + + Not ASA Received None No Change + Discon + + Any STR Received Send STA, Idle + Clean up + + The following state machine is observed by a client when state is not + maintained on the server: + + CLIENT, STATELESS + State Event Action New State + --------------------------------------------------------------- + Idle Client or device requests Send Pending + access service- + specific + auth req + + Pending Successful service-specific Grant Open + authorization answer access + received with Auth-Session- + State set to + NO_STATE_MAINTAINED + + Pending Failed service-specific Clean up Idle + authorization answer + received + + Open Session-Timeout expires on Discon. Idle + access device user/device + + Open Service to user is terminated Discon. Idle + user/device + + + + + + +Fajardo, et al. Standards Track [Page 103] + +RFC 6733 Diameter Base Protocol October 2012 + + + The following state machine is observed by a server when it is not + maintaining state for the session: + + SERVER, STATELESS + State Event Action New State + --------------------------------------------------------------- + Idle Service-specific authorization Send Idle + request received, and service- + successfully processed specific + answer + +8.2. Accounting Session State Machine + + The following state machines MUST be supported for applications that + have an accounting portion or that require only accounting services. + The first state machine is to be observed by clients. + + See Section 9.7 for Accounting Command Codes and Section 9.8 for + Accounting AVPs. + + The server side in the accounting state machine depends in some cases + on the particular application. The Diameter base protocol defines a + default state machine that MUST be followed by all applications that + have not specified other state machines. This is the second state + machine in this section described below. + + The default server side state machine requires the reception of + accounting records in any order and at any time, and it does not + place any standards requirement on the processing of these records. + Implementations of Diameter may perform checking, ordering, + correlation, fraud detection, and other tasks based on these records. + AVPs may need to be inspected as a part of these tasks. The tasks + can happen either immediately after record reception or in a post- + processing phase. However, as these tasks are typically application + or even policy dependent, they are not standardized by the Diameter + specifications. Applications MAY define requirements on when to + accept accounting records based on the used value of Accounting- + Realtime-Required AVP, credit-limit checks, and so on. + + However, the Diameter base protocol defines one optional server side + state machine that MAY be followed by applications that require + keeping track of the session state at the accounting server. Note + that such tracking is incompatible with the ability to sustain long + duration connectivity problems. Therefore, the use of this state + machine is recommended only in applications where the value of the + Accounting-Realtime-Required AVP is DELIVER_AND_GRANT; hence, + accounting connectivity problems are required to cause the serviced + user to be disconnected. Otherwise, records produced by the client + + + +Fajardo, et al. Standards Track [Page 104] + +RFC 6733 Diameter Base Protocol October 2012 + + + may be lost by the server, which no longer accepts them after the + connectivity is re-established. This state machine is the third + state machine in this section. The state machine is supervised by a + supervision session timer Ts, whose value should be reasonably higher + than the Acct_Interim_Interval value. Ts MAY be set to two times the + value of the Acct_Interim_Interval so as to avoid the accounting + session in the Diameter server to change to Idle state in case of + short transient network failure. + + Any event not listed in the state machines MUST be considered as an + error condition, and a corresponding answer, if applicable, MUST be + returned to the originator of the message. + + In the state table, the event "Failure to send" means that the + Diameter client is unable to communicate with the desired + destination. This could be due to the peer being down, or due to the + peer sending back a transient failure or temporary protocol error + notification DIAMETER_OUT_OF_SPACE, DIAMETER_TOO_BUSY, or + DIAMETER_LOOP_DETECTED in the Result-Code AVP of the Accounting + Answer command. + + The event "Failed answer" means that the Diameter client received a + non-transient failure notification in the Accounting Answer command. + + Note that the action "Disconnect user/dev" MUST also have an effect + on the authorization session state table, e.g., cause the STR message + to be sent, if the given application has both authentication/ + authorization and accounting portions. + + The states PendingS, PendingI, PendingL, PendingE, and PendingB stand + for pending states to wait for an answer to an accounting request + related to a Start, Interim, Stop, Event, or buffered record, + respectively. + + CLIENT, ACCOUNTING + State Event Action New State + --------------------------------------------------------------- + Idle Client or device requests Send PendingS + access accounting + start req. + + Idle Client or device requests Send PendingE + a one-time service accounting + event req + + Idle Records in storage Send PendingB + record + + + + +Fajardo, et al. Standards Track [Page 105] + +RFC 6733 Diameter Base Protocol October 2012 + + + PendingS Successful accounting Open + start answer received + + PendingS Failure to send and buffer Store Open + space available and real time Start + not equal to DELIVER_AND_GRANT Record + + PendingS Failure to send and no buffer Open + space available and real time + equal to GRANT_AND_LOSE + + PendingS Failure to send and no Disconnect Idle + buffer space available and user/dev + real time not equal to + GRANT_AND_LOSE + + PendingS Failed accounting start answer Open + received and real time equal + to GRANT_AND_LOSE + + PendingS Failed accounting start answer Disconnect Idle + received and real time not user/dev + equal to GRANT_AND_LOSE + + PendingS User service terminated Store PendingS + stop + record + + Open Interim interval elapses Send PendingI + accounting + interim + record + + Open User service terminated Send PendingL + accounting + stop req. + + PendingI Successful accounting interim Open + answer received + + PendingI Failure to send and (buffer Store Open + space available or old interim + record can be overwritten) record + and real time not equal to + DELIVER_AND_GRANT + + + + + + +Fajardo, et al. Standards Track [Page 106] + +RFC 6733 Diameter Base Protocol October 2012 + + + PendingI Failure to send and no buffer Open + space available and real time + equal to GRANT_AND_LOSE + + PendingI Failure to send and no Disconnect Idle + buffer space available and user/dev + real time not equal to + GRANT_AND_LOSE + + PendingI Failed accounting interim Open + answer received and real time + equal to GRANT_AND_LOSE + + PendingI Failed accounting interim Disconnect Idle + answer received and user/dev + real time not equal to + GRANT_AND_LOSE + + PendingI User service terminated Store PendingI + stop + record + PendingE Successful accounting Idle + event answer received + + PendingE Failure to send and buffer Store Idle + space available event + record + + PendingE Failure to send and no buffer Idle + space available + + PendingE Failed accounting event answer Idle + received + + PendingB Successful accounting answer Delete Idle + received record + + PendingB Failure to send Idle + + PendingB Failed accounting answer Delete Idle + received record + + PendingL Successful accounting Idle + stop answer received + + PendingL Failure to send and buffer Store Idle + space available stop + record + + + +Fajardo, et al. Standards Track [Page 107] + +RFC 6733 Diameter Base Protocol October 2012 + + + PendingL Failure to send and no buffer Idle + space available + + PendingL Failed accounting stop answer Idle + received + + + SERVER, STATELESS ACCOUNTING + State Event Action New State + --------------------------------------------------------------- + + Idle Accounting start request Send Idle + received and successfully accounting + processed. start + answer + + Idle Accounting event request Send Idle + received and successfully accounting + processed. event + answer + + Idle Interim record received Send Idle + and successfully processed. accounting + interim + answer + + Idle Accounting stop request Send Idle + received and successfully accounting + processed stop answer + + Idle Accounting request received; Send Idle + no space left to store accounting + records answer; + Result-Code = + OUT_OF_ + SPACE + + + + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 108] + +RFC 6733 Diameter Base Protocol October 2012 + + + SERVER, STATEFUL ACCOUNTING + State Event Action New State + --------------------------------------------------------------- + + Idle Accounting start request Send Open + received and successfully accounting + processed. start + answer; + Start Ts + + Idle Accounting event request Send Idle + received and successfully accounting + processed. event + answer + Idle Accounting request received; Send Idle + no space left to store accounting + records answer; + Result-Code = + OUT_OF_ + SPACE + + Open Interim record received Send Open + and successfully processed. accounting + interim + answer; + Restart Ts + + Open Accounting stop request Send Idle + received and successfully accounting + processed stop answer; + Stop Ts + + Open Accounting request received; Send Idle + no space left to store accounting + records answer; + Result-Code = + OUT_OF_ + SPACE; + Stop Ts + + Open Session supervision timer Ts Stop Ts Idle + expired + + + + + + + + + +Fajardo, et al. Standards Track [Page 109] + +RFC 6733 Diameter Base Protocol October 2012 + + +8.3. Server-Initiated Re-Auth + + A Diameter server may initiate a re-authentication and/or re- + authorization service for a particular session by issuing a Re-Auth- + Request (RAR). + + For example, for prepaid services, the Diameter server that + originally authorized a session may need some confirmation that the + user is still using the services. + + An access device that receives an RAR message with the Session-Id + equal to a currently active session MUST initiate a re-auth towards + the user, if the service supports this particular feature. Each + Diameter application MUST state whether server-initiated re-auth is + supported, since some applications do not allow access devices to + prompt the user for re-auth. + +8.3.1. Re-Auth-Request + + The Re-Auth-Request (RAR), indicated by the Command Code set to 258 + and the message flags' 'R' bit set, may be sent by any server to the + access device that is providing session service, to request that the + user be re-authenticated and/or re-authorized. + + + Message Format + + <RAR> ::= < Diameter Header: 258, REQ, PXY > + < Session-Id > + { Origin-Host } + { Origin-Realm } + { Destination-Realm } + { Destination-Host } + { Auth-Application-Id } + { Re-Auth-Request-Type } + [ User-Name ] + [ Origin-State-Id ] + * [ Proxy-Info ] + * [ Route-Record ] + * [ AVP ] + +8.3.2. Re-Auth-Answer + + The Re-Auth-Answer (RAA), indicated by the Command Code set to 258 + and the message flags' 'R' bit clear, is sent in response to the RAR. + The Result-Code AVP MUST be present, and it indicates the disposition + of the request. + + + + +Fajardo, et al. Standards Track [Page 110] + +RFC 6733 Diameter Base Protocol October 2012 + + + A successful RAA message MUST be followed by an application-specific + authentication and/or authorization message. + + Message Format + + <RAA> ::= < Diameter Header: 258, PXY > + < Session-Id > + { Result-Code } + { Origin-Host } + { Origin-Realm } + [ User-Name ] + [ Origin-State-Id ] + [ Error-Message ] + [ Error-Reporting-Host ] + [ Failed-AVP ] + * [ Redirect-Host ] + [ Redirect-Host-Usage ] + [ Redirect-Max-Cache-Time ] + * [ Proxy-Info ] + * [ AVP ] + +8.4. Session Termination + + It is necessary for a Diameter server that authorized a session, for + which it is maintaining state, to be notified when that session is no + longer active, both for tracking purposes as well as to allow + stateful agents to release any resources that they may have provided + for the user's session. For sessions whose state is not being + maintained, this section is not used. + + When a user session that required Diameter authorization terminates, + the access device that provided the service MUST issue a Session- + Termination-Request (STR) message to the Diameter server that + authorized the service, to notify it that the session is no longer + active. An STR MUST be issued when a user session terminates for any + reason, including user logoff, expiration of Session-Timeout, + administrative action, termination upon receipt of an Abort-Session- + Request (see below), orderly shutdown of the access device, etc. + + The access device also MUST issue an STR for a session that was + authorized but never actually started. This could occur, for + example, due to a sudden resource shortage in the access device, or + because the access device is unwilling to provide the type of service + requested in the authorization, or because the access device does not + support a mandatory AVP returned in the authorization, etc. + + It is also possible that a session that was authorized is never + actually started due to action of a proxy. For example, a proxy may + + + +Fajardo, et al. Standards Track [Page 111] + +RFC 6733 Diameter Base Protocol October 2012 + + + modify an authorization answer, converting the result from success to + failure, prior to forwarding the message to the access device. If + the answer did not contain an Auth-Session-State AVP with the value + NO_STATE_MAINTAINED, a proxy that causes an authorized session not to + be started MUST issue an STR to the Diameter server that authorized + the session, since the access device has no way of knowing that the + session had been authorized. + + A Diameter server that receives an STR message MUST clean up + resources (e.g., session state) associated with the Session-Id + specified in the STR and return a Session-Termination-Answer. + + A Diameter server also MUST clean up resources when the Session- + Timeout expires, or when the Authorization-Lifetime and the Auth- + Grace-Period AVPs expire without receipt of a re-authorization + request, regardless of whether an STR for that session is received. + The access device is not expected to provide service beyond the + expiration of these timers; thus, expiration of either of these + timers implies that the access device may have unexpectedly shut + down. + +8.4.1. Session-Termination-Request + + The Session-Termination-Request (STR), indicated by the Command Code + set to 275 and the Command Flags' 'R' bit set, is sent by a Diameter + client or by a Diameter proxy to inform the Diameter server that an + authenticated and/or authorized session is being terminated. + + Message Format + + <STR> ::= < Diameter Header: 275, REQ, PXY > + < Session-Id > + { Origin-Host } + { Origin-Realm } + { Destination-Realm } + { Auth-Application-Id } + { Termination-Cause } + [ User-Name ] + [ Destination-Host ] + * [ Class ] + [ Origin-State-Id ] + * [ Proxy-Info ] + * [ Route-Record ] + * [ AVP ] + + + + + + + +Fajardo, et al. Standards Track [Page 112] + +RFC 6733 Diameter Base Protocol October 2012 + + +8.4.2. Session-Termination-Answer + + The Session-Termination-Answer (STA), indicated by the Command Code + set to 275 and the message flags' 'R' bit clear, is sent by the + Diameter server to acknowledge the notification that the session has + been terminated. The Result-Code AVP MUST be present, and it MAY + contain an indication that an error occurred while servicing the STR. + + Upon sending or receipt of the STA, the Diameter server MUST release + all resources for the session indicated by the Session-Id AVP. Any + intermediate server in the Proxy-Chain MAY also release any + resources, if necessary. + + Message Format + + <STA> ::= < Diameter Header: 275, PXY > + < Session-Id > + { Result-Code } + { Origin-Host } + { Origin-Realm } + [ User-Name ] + * [ Class ] + [ Error-Message ] + [ Error-Reporting-Host ] + [ Failed-AVP ] + [ Origin-State-Id ] + * [ Redirect-Host ] + [ Redirect-Host-Usage ] + [ Redirect-Max-Cache-Time ] + * [ Proxy-Info ] + * [ AVP ] + +8.5. Aborting a Session + + A Diameter server may request that the access device stop providing + service for a particular session by issuing an Abort-Session-Request + (ASR). + + For example, the Diameter server that originally authorized the + session may be required to cause that session to be stopped for lack + of credit or other reasons that were not anticipated when the session + was first authorized. + + An access device that receives an ASR with Session-ID equal to a + currently active session MAY stop the session. Whether the access + device stops the session or not is implementation and/or + configuration dependent. For example, an access device may honor + ASRs from certain agents only. In any case, the access device MUST + + + +Fajardo, et al. Standards Track [Page 113] + +RFC 6733 Diameter Base Protocol October 2012 + + + respond with an Abort-Session-Answer, including a Result-Code AVP to + indicate what action it took. + +8.5.1. Abort-Session-Request + + The Abort-Session-Request (ASR), indicated by the Command Code set to + 274 and the message flags' 'R' bit set, may be sent by any Diameter + server or any Diameter proxy to the access device that is providing + session service, to request that the session identified by the + Session-Id be stopped. + + Message Format + + <ASR> ::= < Diameter Header: 274, REQ, PXY > + < Session-Id > + { Origin-Host } + { Origin-Realm } + { Destination-Realm } + { Destination-Host } + { Auth-Application-Id } + [ User-Name ] + [ Origin-State-Id ] + * [ Proxy-Info ] + * [ Route-Record ] + * [ AVP ] + +8.5.2. Abort-Session-Answer + + The Abort-Session-Answer (ASA), indicated by the Command Code set to + 274 and the message flags' 'R' bit clear, is sent in response to the + ASR. The Result-Code AVP MUST be present and indicates the + disposition of the request. + + If the session identified by Session-Id in the ASR was successfully + terminated, the Result-Code is set to DIAMETER_SUCCESS. If the + session is not currently active, the Result-Code is set to + DIAMETER_UNKNOWN_SESSION_ID. If the access device does not stop the + session for any other reason, the Result-Code is set to + DIAMETER_UNABLE_TO_COMPLY. + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 114] + +RFC 6733 Diameter Base Protocol October 2012 + + + Message Format + + <ASA> ::= < Diameter Header: 274, PXY > + < Session-Id > + { Result-Code } + { Origin-Host } + { Origin-Realm } + [ User-Name ] + [ Origin-State-Id ] + [ Error-Message ] + [ Error-Reporting-Host ] + [ Failed-AVP ] + * [ Redirect-Host ] + [ Redirect-Host-Usage ] + [ Redirect-Max-Cache-Time ] + * [ Proxy-Info ] + * [ AVP ] + +8.6. Inferring Session Termination from Origin-State-Id + + The Origin-State-Id is used to allow detection of terminated sessions + for which no STR would have been issued, due to unanticipated + shutdown of an access device. + + A Diameter client or access device increments the value of the + Origin-State-Id every time it is started or powered up. The new + Origin-State-Id is then sent in the CER/CEA message immediately upon + connection to the server. The Diameter server receiving the new + Origin-State-Id can determine whether the sending Diameter client had + abruptly shut down by comparing the old value of the Origin-State-Id + it has kept for that specific client is less than the new value and + whether it has un-terminated sessions originating from that client. + + An access device can also include the Origin-State-Id in request + messages other than the CER if there are relays or proxies in between + the access device and the server. In this case, however, the server + cannot discover that the access device has been restarted unless and + until it receives a new request from it. Therefore, this mechanism + is more opportunistic across proxies and relays. + + The Diameter server may assume that all sessions that were active + prior to detection of a client restart have been terminated. The + Diameter server MAY clean up all session state associated with such + lost sessions, and it MAY also issue STRs for all such lost sessions + that were authorized on upstream servers, to allow session state to + be cleaned up globally. + + + + + +Fajardo, et al. Standards Track [Page 115] + +RFC 6733 Diameter Base Protocol October 2012 + + +8.7. Auth-Request-Type AVP + + The Auth-Request-Type AVP (AVP Code 274) is of type Enumerated and is + included in application-specific auth requests to inform the peers + whether a user is to be authenticated only, authorized only, or both. + Note any value other than both MAY cause RADIUS interoperability + issues. The following values are defined: + + AUTHENTICATE_ONLY 1 + + The request being sent is for authentication only, and it MUST + contain the relevant application-specific authentication AVPs that + are needed by the Diameter server to authenticate the user. + + AUTHORIZE_ONLY 2 + + The request being sent is for authorization only, and it MUST + contain the application-specific authorization AVPs that are + necessary to identify the service being requested/offered. + + AUTHORIZE_AUTHENTICATE 3 + + The request contains a request for both authentication and + authorization. The request MUST include both the relevant + application-specific authentication information and authorization + information necessary to identify the service being requested/ + offered. + +8.8. Session-Id AVP + + The Session-Id AVP (AVP Code 263) is of type UTF8String and is used + to identify a specific session (see Section 8). All messages + pertaining to a specific session MUST include only one Session-Id + AVP, and the same value MUST be used throughout the life of a + session. When present, the Session-Id SHOULD appear immediately + following the Diameter header (see Section 3). + + The Session-Id MUST be globally and eternally unique, as it is meant + to uniquely identify a user session without reference to any other + information, and it may be needed to correlate historical + authentication information with accounting information. The + Session-Id includes a mandatory portion and an implementation-defined + portion; a recommended format for the implementation-defined portion + is outlined below. + + The Session-Id MUST begin with the sender's identity encoded in the + DiameterIdentity type (see Section 4.3.1). The remainder of the + Session-Id is delimited by a ";" character, and it MAY be any + + + +Fajardo, et al. Standards Track [Page 116] + +RFC 6733 Diameter Base Protocol October 2012 + + + sequence that the client can guarantee to be eternally unique; + however, the following format is recommended, (square brackets [] + indicate an optional element): + + <DiameterIdentity>;<high 32 bits>;<low 32 bits>[;<optional value>] + + <high 32 bits> and <low 32 bits> are decimal representations of the + high and low 32 bits of a monotonically increasing 64-bit value. The + 64-bit value is rendered in two part to simplify formatting by 32-bit + processors. At startup, the high 32 bits of the 64-bit value MAY be + initialized to the time in NTP format [RFC5905], and the low 32 bits + MAY be initialized to zero. This will for practical purposes + eliminate the possibility of overlapping Session-Ids after a reboot, + assuming the reboot process takes longer than a second. + Alternatively, an implementation MAY keep track of the increasing + value in non-volatile memory. + + + <optional value> is implementation specific, but it may include a + modem's device Id, a Layer 2 address, timestamp, etc. + + Example, in which there is no optional value: + + accesspoint7.example.com;1876543210;523 + + Example, in which there is an optional value: + + accesspoint7.example.com;1876543210;523;[email protected] + + The Session-Id is created by the Diameter application initiating the + session, which, in most cases, is done by the client. Note that a + Session-Id MAY be used for both the authentication, authorization, + and accounting commands of a given application. + +8.9. Authorization-Lifetime AVP + + The Authorization-Lifetime AVP (AVP Code 291) is of type Unsigned32 + and contains the maximum number of seconds of service to be provided + to the user before the user is to be re-authenticated and/or re- + authorized. Care should be taken when the Authorization-Lifetime + value is determined, since a low, non-zero value could create + significant Diameter traffic, which could congest both the network + and the agents. + + A value of zero (0) means that immediate re-auth is necessary by the + access device. The absence of this AVP, or a value of all ones + (meaning all bits in the 32-bit field are set to one) means no re- + auth is expected. + + + +Fajardo, et al. Standards Track [Page 117] + +RFC 6733 Diameter Base Protocol October 2012 + + + If both this AVP and the Session-Timeout AVP are present in a + message, the value of the latter MUST NOT be smaller than the + Authorization-Lifetime AVP. + + An Authorization-Lifetime AVP MAY be present in re-authorization + messages, and it contains the number of seconds the user is + authorized to receive service from the time the re-auth answer + message is received by the access device. + + This AVP MAY be provided by the client as a hint of the maximum + lifetime that it is willing to accept. The server MUST return a + value that is equal to, or smaller than, the one provided by the + client. + +8.10. Auth-Grace-Period AVP + + The Auth-Grace-Period AVP (AVP Code 276) is of type Unsigned32 and + contains the number of seconds the Diameter server will wait + following the expiration of the Authorization-Lifetime AVP before + cleaning up resources for the session. + +8.11. Auth-Session-State AVP + + The Auth-Session-State AVP (AVP Code 277) is of type Enumerated and + specifies whether state is maintained for a particular session. The + client MAY include this AVP in requests as a hint to the server, but + the value in the server's answer message is binding. The following + values are supported: + + STATE_MAINTAINED 0 + + This value is used to specify that session state is being + maintained, and the access device MUST issue a session termination + message when service to the user is terminated. This is the + default value. + + NO_STATE_MAINTAINED 1 + + This value is used to specify that no session termination messages + will be sent by the access device upon expiration of the + Authorization-Lifetime. + +8.12. Re-Auth-Request-Type AVP + + The Re-Auth-Request-Type AVP (AVP Code 285) is of type Enumerated and + is included in application-specific auth answers to inform the client + of the action expected upon expiration of the Authorization-Lifetime. + + + + +Fajardo, et al. Standards Track [Page 118] + +RFC 6733 Diameter Base Protocol October 2012 + + + If the answer message contains an Authorization-Lifetime AVP with a + positive value, the Re-Auth-Request-Type AVP MUST be present in an + answer message. The following values are defined: + + AUTHORIZE_ONLY 0 + + An authorization only re-auth is expected upon expiration of the + Authorization-Lifetime. This is the default value if the AVP is + not present in answer messages that include the Authorization- + Lifetime. + + AUTHORIZE_AUTHENTICATE 1 + + An authentication and authorization re-auth is expected upon + expiration of the Authorization-Lifetime. + +8.13. Session-Timeout AVP + + The Session-Timeout AVP (AVP Code 27) [RFC2865] is of type Unsigned32 + and contains the maximum number of seconds of service to be provided + to the user before termination of the session. When both the + Session-Timeout and the Authorization-Lifetime AVPs are present in an + answer message, the former MUST be equal to or greater than the value + of the latter. + + A session that terminates on an access device due to the expiration + of the Session-Timeout MUST cause an STR to be issued, unless both + the access device and the home server had previously agreed that no + session termination messages would be sent (see Section 8). + + A Session-Timeout AVP MAY be present in a re-authorization answer + message, and it contains the remaining number of seconds from the + beginning of the re-auth. + + A value of zero, or the absence of this AVP, means that this session + has an unlimited number of seconds before termination. + + This AVP MAY be provided by the client as a hint of the maximum + timeout that it is willing to accept. However, the server MAY return + a value that is equal to, or smaller than, the one provided by the + client. + +8.14. User-Name AVP + + The User-Name AVP (AVP Code 1) [RFC2865] is of type UTF8String, which + contains the User-Name, in a format consistent with the NAI + specification [RFC4282]. + + + + +Fajardo, et al. Standards Track [Page 119] + +RFC 6733 Diameter Base Protocol October 2012 + + +8.15. Termination-Cause AVP + + The Termination-Cause AVP (AVP Code 295) is of type Enumerated, and + is used to indicate the reason why a session was terminated on the + access device. The currently assigned values for this AVP can be + found in the IANA registry for Termination-Cause AVP Values + [IANATCV]. + +8.16. Origin-State-Id AVP + + The Origin-State-Id AVP (AVP Code 278), of type Unsigned32, is a + monotonically increasing value that is advanced whenever a Diameter + entity restarts with loss of previous state, for example, upon + reboot. Origin-State-Id MAY be included in any Diameter message, + including CER. + + A Diameter entity issuing this AVP MUST create a higher value for + this AVP each time its state is reset. A Diameter entity MAY set + Origin-State-Id to the time of startup, or it MAY use an incrementing + counter retained in non-volatile memory across restarts. + + The Origin-State-Id, if present, MUST reflect the state of the entity + indicated by Origin-Host. If a proxy modifies Origin-Host, it MUST + either remove Origin-State-Id or modify it appropriately as well. + Typically, Origin-State-Id is used by an access device that always + starts up with no active sessions; that is, any session active prior + to restart will have been lost. By including Origin-State-Id in a + message, it allows other Diameter entities to infer that sessions + associated with a lower Origin-State-Id are no longer active. If an + access device does not intend for such inferences to be made, it MUST + either not include Origin-State-Id in any message or set its value to + 0. + +8.17. Session-Binding AVP + + The Session-Binding AVP (AVP Code 270) is of type Unsigned32, and it + MAY be present in application-specific authorization answer messages. + If present, this AVP MAY inform the Diameter client that all future + application-specific re-auth and Session-Termination-Request messages + for this session MUST be sent to the same authorization server. + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 120] + +RFC 6733 Diameter Base Protocol October 2012 + + + This field is a bit mask, and the following bits have been defined: + + RE_AUTH 1 + + When set, future re-auth messages for this session MUST NOT + include the Destination-Host AVP. When cleared, the default + value, the Destination-Host AVP MUST be present in all re-auth + messages for this session. + + STR 2 + + When set, the STR message for this session MUST NOT include the + Destination-Host AVP. When cleared, the default value, the + Destination-Host AVP MUST be present in the STR message for this + session. + + ACCOUNTING 4 + + When set, all accounting messages for this session MUST NOT + include the Destination-Host AVP. When cleared, the default + value, the Destination-Host AVP, if known, MUST be present in all + accounting messages for this session. + +8.18. Session-Server-Failover AVP + + The Session-Server-Failover AVP (AVP Code 271) is of type Enumerated + and MAY be present in application-specific authorization answer + messages that either do not include the Session-Binding AVP or + include the Session-Binding AVP with any of the bits set to a zero + value. If present, this AVP MAY inform the Diameter client that if a + re-auth or STR message fails due to a delivery problem, the Diameter + client SHOULD issue a subsequent message without the Destination-Host + AVP. When absent, the default value is REFUSE_SERVICE. + + The following values are supported: + + REFUSE_SERVICE 0 + + If either the re-auth or the STR message delivery fails, terminate + service with the user and do not attempt any subsequent attempts. + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 121] + +RFC 6733 Diameter Base Protocol October 2012 + + + TRY_AGAIN 1 + + If either the re-auth or the STR message delivery fails, resend + the failed message without the Destination-Host AVP present. + + ALLOW_SERVICE 2 + + If re-auth message delivery fails, assume that re-authorization + succeeded. If STR message delivery fails, terminate the session. + + TRY_AGAIN_ALLOW_SERVICE 3 + + If either the re-auth or the STR message delivery fails, resend + the failed message without the Destination-Host AVP present. If + the second delivery fails for re-auth, assume re-authorization + succeeded. If the second delivery fails for STR, terminate the + session. + +8.19. Multi-Round-Time-Out AVP + + The Multi-Round-Time-Out AVP (AVP Code 272) is of type Unsigned32 and + SHOULD be present in application-specific authorization answer + messages whose Result-Code AVP is set to DIAMETER_MULTI_ROUND_AUTH. + This AVP contains the maximum number of seconds that the access + device MUST provide the user in responding to an authentication + request. + +8.20. Class AVP + + The Class AVP (AVP Code 25) is of type OctetString and is used by + Diameter servers to return state information to the access device. + When one or more Class AVPs are present in application-specific + authorization answer messages, they MUST be present in subsequent re- + authorization, session termination and accounting messages. Class + AVPs found in a re-authorization answer message override the ones + found in any previous authorization answer message. Diameter server + implementations SHOULD NOT return Class AVPs that require more than + 4096 bytes of storage on the Diameter client. A Diameter client that + receives Class AVPs whose size exceeds local available storage MUST + terminate the session. + +8.21. Event-Timestamp AVP + + The Event-Timestamp (AVP Code 55) is of type Time and MAY be included + in an Accounting-Request and Accounting-Answer messages to record the + time that the reported event occurred, in seconds since January 1, + 1900 00:00 UTC. + + + + +Fajardo, et al. Standards Track [Page 122] + +RFC 6733 Diameter Base Protocol October 2012 + + +9. Accounting + + This accounting protocol is based on a server directed model with + capabilities for real-time delivery of accounting information. + Several fault resilience methods [RFC2975] have been built into the + protocol in order minimize loss of accounting data in various fault + situations and under different assumptions about the capabilities of + the used devices. + +9.1. Server Directed Model + + The server directed model means that the device generating the + accounting data gets information from either the authorization server + (if contacted) or the accounting server regarding the way accounting + data shall be forwarded. This information includes accounting record + timeliness requirements. + + As discussed in [RFC2975], real-time transfer of accounting records + is a requirement, such as the need to perform credit-limit checks and + fraud detection. Note that batch accounting is not a requirement, + and is therefore not supported by Diameter. Should batched + accounting be required in the future, a new Diameter application will + need to be created, or it could be handled using another protocol. + Note, however, that even if at the Diameter layer, accounting + requests are processed one by one; transport protocols used under + Diameter typically batch several requests in the same packet under + heavy traffic conditions. This may be sufficient for many + applications. + + The authorization server (chain) directs the selection of proper + transfer strategy, based on its knowledge of the user and + relationships of roaming partnerships. The server (or agents) uses + the Acct-Interim-Interval and Accounting-Realtime-Required AVPs to + control the operation of the Diameter peer operating as a client. + The Acct-Interim-Interval AVP, when present, instructs the Diameter + node acting as a client to produce accounting records continuously + even during a session. Accounting-Realtime-Required AVP is used to + control the behavior of the client when the transfer of accounting + records from the Diameter client is delayed or unsuccessful. + + The Diameter accounting server MAY override the interim interval or + the real-time requirements by including the Acct-Interim-Interval or + Accounting-Realtime-Required AVP in the Accounting-Answer message. + When one of these AVPs is present, the latest value received SHOULD + be used in further accounting activities for the same session. + + + + + + +Fajardo, et al. Standards Track [Page 123] + +RFC 6733 Diameter Base Protocol October 2012 + + +9.2. Protocol Messages + + A Diameter node that receives a successful authentication and/or + authorization message from the Diameter server SHOULD collect + accounting information for the session. The Accounting-Request + message is used to transmit the accounting information to the + Diameter server, which MUST reply with the Accounting-Answer message + to confirm reception. The Accounting-Answer message includes the + Result-Code AVP, which MAY indicate that an error was present in the + accounting message. The value of the Accounting-Realtime-Required + AVP received earlier for the session in question may indicate that + the user's session has to be terminated when a rejected Accounting- + Request message was received. + +9.3. Accounting Application Extension and Requirements + + Each Diameter application (e.g., NASREQ, Mobile IP) SHOULD define its + service-specific AVPs that MUST be present in the Accounting-Request + message in a section titled "Accounting AVPs". The application MUST + assume that the AVPs described in this document will be present in + all Accounting messages, so only their respective service-specific + AVPs need to be defined in that section. + + Applications have the option of using one or both of the following + accounting application extension models: + + Split Accounting Service + + The accounting message will carry the Application Id of the + Diameter base accounting application (see Section 2.4). + Accounting messages may be routed to Diameter nodes other than the + corresponding Diameter application. These nodes might be + centralized accounting servers that provide accounting service for + multiple different Diameter applications. These nodes MUST + advertise the Diameter base accounting Application Id during + capabilities exchange. + + Coupled Accounting Service + + The accounting message will carry the Application Id of the + application that is using it. The application itself will process + the received accounting records or forward them to an accounting + server. There is no accounting application advertisement required + during capabilities exchange, and the accounting messages will be + routed the same way as any of the other application messages. + + In cases where an application does not define its own accounting + service, it is preferred that the split accounting model be used. + + + +Fajardo, et al. Standards Track [Page 124] + +RFC 6733 Diameter Base Protocol October 2012 + + +9.4. Fault Resilience + + Diameter base protocol mechanisms are used to overcome small message + loss and network faults of a temporary nature. + + Diameter peers acting as clients MUST implement the use of failover + to guard against server failures and certain network failures. + Diameter peers acting as agents or related off-line processing + systems MUST detect duplicate accounting records caused by the + sending of the same record to several servers and duplication of + messages in transit. This detection MUST be based on the inspection + of the Session-Id and Accounting-Record-Number AVP pairs. Appendix C + discusses duplicate detection needs and implementation issues. + + Diameter clients MAY have non-volatile memory for the safe storage of + accounting records over reboots or extended network failures, network + partitions, and server failures. If such memory is available, the + client SHOULD store new accounting records there as soon as the + records are created and until a positive acknowledgement of their + reception from the Diameter server has been received. Upon a reboot, + the client MUST start sending the records in the non-volatile memory + to the accounting server with the appropriate modifications in + termination cause, session length, and other relevant information in + the records. + + A further application of this protocol may include AVPs to control + the maximum number of accounting records that may be stored in the + Diameter client without committing them to the non-volatile memory or + transferring them to the Diameter server. + + The client SHOULD NOT remove the accounting data from any of its + memory areas before the correct Accounting-Answer has been received. + The client MAY remove the oldest, undelivered, or as yet + unacknowledged accounting data if it runs out of resources such as + memory. It is an implementation-dependent matter for the client to + accept new sessions under this condition. + +9.5. Accounting Records + + In all accounting records, the Session-Id AVP MUST be present; the + User-Name AVP MUST be present if it is available to the Diameter + client. + + Different types of accounting records are sent depending on the + actual type of accounted service and the authorization server's + directions for interim accounting. If the accounted service is a + + + + + +Fajardo, et al. Standards Track [Page 125] + +RFC 6733 Diameter Base Protocol October 2012 + + + one-time event, meaning that the start and stop of the event are + simultaneous, then the Accounting-Record-Type AVP MUST be present and + set to the value EVENT_RECORD. + + If the accounted service is of a measurable length, then the AVP MUST + use the values START_RECORD, STOP_RECORD, and possibly, + INTERIM_RECORD. If the authorization server has not directed interim + accounting to be enabled for the session, two accounting records MUST + be generated for each service of type session. When the initial + Accounting-Request for a given session is sent, the Accounting- + Record-Type AVP MUST be set to the value START_RECORD. When the last + Accounting-Request is sent, the value MUST be STOP_RECORD. + + If the authorization server has directed interim accounting to be + enabled, the Diameter client MUST produce additional records between + the START_RECORD and STOP_RECORD, marked INTERIM_RECORD. The + production of these records is directed by Acct-Interim-Interval as + well as any re-authentication or re-authorization of the session. + The Diameter client MUST overwrite any previous interim accounting + records that are locally stored for delivery, if a new record is + being generated for the same session. This ensures that only one + pending interim record can exist on an access device for any given + session. + + A particular value of Accounting-Sub-Session-Id MUST appear only in + one sequence of accounting records from a Diameter client, except for + the purposes of retransmission. The one sequence that is sent MUST + be either one record with Accounting-Record-Type AVP set to the value + EVENT_RECORD or several records starting with one having the value + START_RECORD, followed by zero or more INTERIM_RECORDs and a single + STOP_RECORD. A particular Diameter application specification MUST + define the type of sequences that MUST be used. + +9.6. Correlation of Accounting Records + + If an application uses accounting messages, it can correlate + accounting records with a specific application session by using the + Session-Id of the particular application session in the accounting + messages. Accounting messages MAY also use a different Session-Id + from that of the application sessions, in which case, other session- + related information is needed to perform correlation. + + In cases where an application requires multiple accounting sub- + sessions, an Accounting-Sub-Session-Id AVP is used to differentiate + each sub-session. The Session-Id would remain constant for all sub- + sessions and is used to correlate all the sub-sessions to a + particular application session. Note that receiving a STOP_RECORD + + + + +Fajardo, et al. Standards Track [Page 126] + +RFC 6733 Diameter Base Protocol October 2012 + + + with no Accounting-Sub-Session-Id AVP when sub-sessions were + originally used in the START_RECORD messages implies that all sub- + sessions are terminated. + + There are also cases where an application needs to correlate multiple + application sessions into a single accounting record; the accounting + record may span multiple different Diameter applications and sessions + used by the same user at a given time. In such cases, the Acct- + Multi-Session-Id AVP is used. The Acct-Multi-Session-Id AVP SHOULD + be signaled by the server to the access device (typically, during + authorization) when it determines that a request belongs to an + existing session. The access device MUST then include the Acct- + Multi-Session-Id AVP in all subsequent accounting messages. + + The Acct-Multi-Session-Id AVP MAY include the value of the original + Session-Id. Its contents are implementation specific, but the MUST + be globally unique across other Acct-Multi-Session-Ids and MUST NOT + change during the life of a session. + + A Diameter application document MUST define the exact concept of a + session that is being accounted, and it MAY define the concept of a + multi-session. For instance, the NASREQ DIAMETER application treats + a single PPP connection to a Network Access Server as one session and + a set of Multilink PPP sessions as one multi-session. + +9.7. Accounting Command Codes + + This section defines Command Code values that MUST be supported by + all Diameter implementations that provide accounting services. + +9.7.1. Accounting-Request + + The Accounting-Request (ACR) command, indicated by the Command Code + field set to 271 and the Command Flags' 'R' bit set, is sent by a + Diameter node, acting as a client, in order to exchange accounting + information with a peer. + + In addition to the AVPs listed below, Accounting-Request messages + SHOULD include service-specific accounting AVPs. + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 127] + +RFC 6733 Diameter Base Protocol October 2012 + + + Message Format + + <ACR> ::= < Diameter Header: 271, REQ, PXY > + < Session-Id > + { Origin-Host } + { Origin-Realm } + { Destination-Realm } + { Accounting-Record-Type } + { Accounting-Record-Number } + [ Acct-Application-Id ] + [ Vendor-Specific-Application-Id ] + [ User-Name ] + [ Destination-Host ] + [ Accounting-Sub-Session-Id ] + [ Acct-Session-Id ] + [ Acct-Multi-Session-Id ] + [ Acct-Interim-Interval ] + [ Accounting-Realtime-Required ] + [ Origin-State-Id ] + [ Event-Timestamp ] + * [ Proxy-Info ] + * [ Route-Record ] + * [ AVP ] + +9.7.2. Accounting-Answer + + The Accounting-Answer (ACA) command, indicated by the Command Code + field set to 271 and the Command Flags' 'R' bit cleared, is used to + acknowledge an Accounting-Request command. The Accounting-Answer + command contains the same Session-Id as the corresponding request. + + Only the target Diameter server, known as the home Diameter server, + SHOULD respond with the Accounting-Answer command. + + In addition to the AVPs listed below, Accounting-Answer messages + SHOULD include service-specific accounting AVPs. + + + + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 128] + +RFC 6733 Diameter Base Protocol October 2012 + + + Message Format + + <ACA> ::= < Diameter Header: 271, PXY > + < Session-Id > + { Result-Code } + { Origin-Host } + { Origin-Realm } + { Accounting-Record-Type } + { Accounting-Record-Number } + [ Acct-Application-Id ] + [ Vendor-Specific-Application-Id ] + [ User-Name ] + [ Accounting-Sub-Session-Id ] + [ Acct-Session-Id ] + [ Acct-Multi-Session-Id ] + [ Error-Message ] + [ Error-Reporting-Host ] + [ Failed-AVP ] + [ Acct-Interim-Interval ] + [ Accounting-Realtime-Required ] + [ Origin-State-Id ] + [ Event-Timestamp ] + * [ Proxy-Info ] + * [ AVP ] + +9.8. Accounting AVPs + + This section contains AVPs that describe accounting usage information + related to a specific session. + +9.8.1. Accounting-Record-Type AVP + + The Accounting-Record-Type AVP (AVP Code 480) is of type Enumerated + and contains the type of accounting record being sent. The following + values are currently defined for the Accounting-Record-Type AVP: + + EVENT_RECORD 1 + + An Accounting Event Record is used to indicate that a one-time + event has occurred (meaning that the start and end of the event + are simultaneous). This record contains all information relevant + to the service, and it is the only record of the service. + + + + + + + + + +Fajardo, et al. Standards Track [Page 129] + +RFC 6733 Diameter Base Protocol October 2012 + + + START_RECORD 2 + + Accounting Start, Interim, and Stop Records are used to indicate + that a service of a measurable length has been given. An + Accounting Start Record is used to initiate an accounting session + and contains accounting information that is relevant to the + initiation of the session. + + INTERIM_RECORD 3 + + An Interim Accounting Record contains cumulative accounting + information for an existing accounting session. Interim + Accounting Records SHOULD be sent every time a re-authentication + or re-authorization occurs. Further, additional interim record + triggers MAY be defined by application-specific Diameter + applications. The selection of whether to use INTERIM_RECORD + records is done by the Acct-Interim-Interval AVP. + + STOP_RECORD 4 + + An Accounting Stop Record is sent to terminate an accounting + session and contains cumulative accounting information relevant to + the existing session. + +9.8.2. Acct-Interim-Interval AVP + + The Acct-Interim-Interval AVP (AVP Code 85) is of type Unsigned32 and + is sent from the Diameter home authorization server to the Diameter + client. The client uses information in this AVP to decide how and + when to produce accounting records. With different values in this + AVP, service sessions can result in one, two, or two+N accounting + records, based on the needs of the home organization. The following + accounting record production behavior is directed by the inclusion of + this AVP: + + 1. The omission of the Acct-Interim-Interval AVP or its inclusion + with Value field set to 0 means that EVENT_RECORD, START_RECORD, + and STOP_RECORD are produced, as appropriate for the service. + + 2. The inclusion of the AVP with Value field set to a non-zero value + means that INTERIM_RECORD records MUST be produced between the + START_RECORD and STOP_RECORD records. The Value field of this + AVP is the nominal interval between these records in seconds. + The Diameter node that originates the accounting information, + known as the client, MUST produce the first INTERIM_RECORD record + roughly at the time when this nominal interval has elapsed from + + + + + +Fajardo, et al. Standards Track [Page 130] + +RFC 6733 Diameter Base Protocol October 2012 + + + the START_RECORD, the next one again as the interval has elapsed + once more, and so on until the session ends and a STOP_RECORD + record is produced. + + The client MUST ensure that the interim record production times + are randomized so that large accounting message storms are not + created either among records or around a common service start + time. + +9.8.3. Accounting-Record-Number AVP + + The Accounting-Record-Number AVP (AVP Code 485) is of type Unsigned32 + and identifies this record within one session. As Session-Id AVPs + are globally unique, the combination of Session-Id and Accounting- + Record-Number AVPs is also globally unique and can be used in + matching accounting records with confirmations. An easy way to + produce unique numbers is to set the value to 0 for records of type + EVENT_RECORD and START_RECORD and set the value to 1 for the first + INTERIM_RECORD, 2 for the second, and so on until the value for + STOP_RECORD is one more than for the last INTERIM_RECORD. + +9.8.4. Acct-Session-Id AVP + + The Acct-Session-Id AVP (AVP Code 44) is of type OctetString is only + used when RADIUS/Diameter translation occurs. This AVP contains the + contents of the RADIUS Acct-Session-Id attribute. + +9.8.5. Acct-Multi-Session-Id AVP + + The Acct-Multi-Session-Id AVP (AVP Code 50) is of type UTF8String, + following the format specified in Section 8.8. The Acct-Multi- + Session-Id AVP is used to link multiple related accounting sessions, + where each session would have a unique Session-Id but the same Acct- + Multi-Session-Id AVP. This AVP MAY be returned by the Diameter + server in an authorization answer, and it MUST be used in all + accounting messages for the given session. + +9.8.6. Accounting-Sub-Session-Id AVP + + The Accounting-Sub-Session-Id AVP (AVP Code 287) is of type + Unsigned64 and contains the accounting sub-session identifier. The + combination of the Session-Id and this AVP MUST be unique per sub- + session, and the value of this AVP MUST be monotonically increased by + one for all new sub-sessions. The absence of this AVP implies no + sub-sessions are in use, with the exception of an Accounting-Request + whose Accounting-Record-Type is set to STOP_RECORD. A STOP_RECORD + message with no Accounting-Sub-Session-Id AVP present will signal the + termination of all sub-sessions for a given Session-Id. + + + +Fajardo, et al. Standards Track [Page 131] + +RFC 6733 Diameter Base Protocol October 2012 + + +9.8.7. Accounting-Realtime-Required AVP + + The Accounting-Realtime-Required AVP (AVP Code 483) is of type + Enumerated and is sent from the Diameter home authorization server to + the Diameter client or in the Accounting-Answer from the accounting + server. The client uses information in this AVP to decide what to do + if the sending of accounting records to the accounting server has + been temporarily prevented due to, for instance, a network problem. + + DELIVER_AND_GRANT 1 + + The AVP with Value field set to DELIVER_AND_GRANT means that the + service MUST only be granted as long as there is a connection to + an accounting server. Note that the set of alternative accounting + servers are treated as one server in this sense. Having to move + the accounting record stream to a backup server is not a reason to + discontinue the service to the user. + + GRANT_AND_STORE 2 + + The AVP with Value field set to GRANT_AND_STORE means that service + SHOULD be granted if there is a connection, or as long as records + can still be stored as described in Section 9.4. + + This is the default behavior if the AVP isn't included in the + reply from the authorization server. + + GRANT_AND_LOSE 3 + + The AVP with Value field set to GRANT_AND_LOSE means that service + SHOULD be granted even if the records cannot be delivered or + stored. + +10. AVP Occurrence Tables + + The following tables present the AVPs defined in this document and + specify in which Diameter messages they MAY or MAY NOT be present. + AVPs that occur only inside a Grouped AVP are not shown in these + tables. + + The tables use the following symbols: + + 0 The AVP MUST NOT be present in the message. + + 0+ Zero or more instances of the AVP MAY be present in the + message. + + + + + +Fajardo, et al. Standards Track [Page 132] + +RFC 6733 Diameter Base Protocol October 2012 + + + 0-1 Zero or one instance of the AVP MAY be present in the message. + It is considered an error if there are more than one instance + of the AVP. + + 1 One instance of the AVP MUST be present in the message. + + 1+ At least one instance of the AVP MUST be present in the + message. + +10.1. Base Protocol Command AVP Table + + The table in this section is limited to the non-Accounting Command + Codes defined in this specification. + + +-----------------------------------------------+ + | Command Code | + +---+---+---+---+---+---+---+---+---+---+---+---+ + Attribute Name |CER|CEA|DPR|DPA|DWR|DWA|RAR|RAA|ASR|ASA|STR|STA| + --------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + Acct-Interim- |0 |0 |0 |0 |0 |0 |0-1|0 |0 |0 |0 |0 | + Interval | | | | | | | | | | | | | + Accounting-Realtime-|0 |0 |0 |0 |0 |0 |0-1|0 |0 |0 |0 |0 | + Required | | | | | | | | | | | | | + Acct-Application-Id |0+ |0+ |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Auth-Application-Id |0+ |0+ |0 |0 |0 |0 |1 |0 |1 |0 |1 |0 | + Auth-Grace-Period |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Auth-Request-Type |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Auth-Session-State |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Authorization- |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Lifetime | | | | | | | | | | | | | + Class |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0+ |0+ | + Destination-Host |0 |0 |0 |0 |0 |0 |1 |0 |1 |0 |0-1|0 | + Destination-Realm |0 |0 |0 |0 |0 |0 |1 |0 |1 |0 |1 |0 | + Disconnect-Cause |0 |0 |1 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Error-Message |0 |0-1|0 |0-1|0 |0-1|0 |0-1|0 |0-1|0 |0-1| + Error-Reporting-Host|0 |0 |0 |0 |0 |0 |0 |0-1|0 |0-1|0 |0-1| + Failed-AVP |0 |0-1|0 |0-1|0 |0-1|0 |0-1|0 |0-1|0 |0-1| + Firmware-Revision |0-1|0-1|0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Host-IP-Address |1+ |1+ |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Inband-Security-Id |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Multi-Round-Time-Out|0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + + + + + + + + + + +Fajardo, et al. Standards Track [Page 133] + +RFC 6733 Diameter Base Protocol October 2012 + + + Origin-Host |1 |1 |1 |1 |1 |1 |1 |1 |1 |1 |1 |1 | + Origin-Realm |1 |1 |1 |1 |1 |1 |1 |1 |1 |1 |1 |1 | + Origin-State-Id |0-1|0-1|0 |0 |0-1|0-1|0-1|0-1|0-1|0-1|0-1|0-1| + Product-Name |1 |1 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Proxy-Info |0 |0 |0 |0 |0 |0 |0+ |0+ |0+ |0+ |0+ |0+ | + Redirect-Host |0 |0 |0 |0 |0 |0 |0 |0+ |0 |0+ |0 |0+ | + Redirect-Host-Usage |0 |0 |0 |0 |0 |0 |0 |0-1|0 |0-1|0 |0-1| + Redirect-Max-Cache- |0 |0 |0 |0 |0 |0 |0 |0-1|0 |0-1|0 |0-1| + Time | | | | | | | | | | | | | + Result-Code |0 |1 |0 |1 |0 |1 |0 |1 |0 |1 |0 |1 | + Re-Auth-Request-Type|0 |0 |0 |0 |0 |0 |1 |0 |0 |0 |0 |0 | + Route-Record |0 |0 |0 |0 |0 |0 |0+ |0 |0+ |0 |0+ |0 | + Session-Binding |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Session-Id |0 |0 |0 |0 |0 |0 |1 |1 |1 |1 |1 |1 | + Session-Server- |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Failover | | | | | | | | | | | | | + Session-Timeout |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Supported-Vendor-Id |0+ |0+ |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Termination-Cause |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 |1 |0 | + User-Name |0 |0 |0 |0 |0 |0 |0-1|0-1|0-1|0-1|0-1|0-1| + Vendor-Id |1 |1 |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Vendor-Specific- |0+ |0+ |0 |0 |0 |0 |0 |0 |0 |0 |0 |0 | + Application-Id | | | | | | | | | | | | | + --------------------+---+---+---+---+---+---+---+---+---+---+---+---+ + +10.2. Accounting AVP Table + + The table in this section is used to represent which AVPs defined in + this document are to be present in the Accounting messages. These + AVP occurrence requirements are guidelines, which may be expanded, + and/or overridden by application-specific requirements in the + Diameter applications documents. + + + + + + + + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 134] + +RFC 6733 Diameter Base Protocol October 2012 + + + +-----------+ + | Command | + | Code | + +-----+-----+ + Attribute Name | ACR | ACA | + ------------------------------+-----+-----+ + Acct-Interim-Interval | 0-1 | 0-1 | + Acct-Multi-Session-Id | 0-1 | 0-1 | + Accounting-Record-Number | 1 | 1 | + Accounting-Record-Type | 1 | 1 | + Acct-Session-Id | 0-1 | 0-1 | + Accounting-Sub-Session-Id | 0-1 | 0-1 | + Accounting-Realtime-Required | 0-1 | 0-1 | + Acct-Application-Id | 0-1 | 0-1 | + Auth-Application-Id | 0 | 0 | + Class | 0+ | 0+ | + Destination-Host | 0-1 | 0 | + Destination-Realm | 1 | 0 | + Error-Reporting-Host | 0 | 0+ | + Event-Timestamp | 0-1 | 0-1 | + Failed-AVP | 0 | 0-1 | + Origin-Host | 1 | 1 | + Origin-Realm | 1 | 1 | + Proxy-Info | 0+ | 0+ | + Route-Record | 0+ | 0 | + Result-Code | 0 | 1 | + Session-Id | 1 | 1 | + Termination-Cause | 0 | 0 | + User-Name | 0-1 | 0-1 | + Vendor-Specific-Application-Id| 0-1 | 0-1 | + ------------------------------+-----+-----+ + +11. IANA Considerations + + This section provides guidance to the Internet Assigned Numbers + Authority (IANA) regarding registration of values related to the + Diameter protocol, in accordance with [RFC5226]. Existing IANA + registries and assignments put in place by RFC 3588 remain the same + unless explicitly updated or deprecated in this section. + +11.1. AVP Header + + As defined in Section 4, the AVP header contains three fields that + require IANA namespace management: the AVP Code, Vendor-ID, and Flags + fields. + + + + + + +Fajardo, et al. Standards Track [Page 135] + +RFC 6733 Diameter Base Protocol October 2012 + + +11.1.1. AVP Codes + + There are multiple namespaces. Vendors can have their own AVP Codes + namespace that will be identified by their Vendor-ID (also known as + Enterprise-Number), and they control the assignments of their vendor- + specific AVP Codes within their own namespace. The absence of a + Vendor-ID or a Vendor-ID value of zero (0) identifies the IETF AVP + Codes namespace, which is under IANA control. The AVP Codes and + sometimes possible values in an AVP are controlled and maintained by + IANA. AVP Code 0 is not used. AVP Codes 1-255 are managed + separately as RADIUS Attribute Types. Where a Vendor-Specific AVP is + implemented by more than one vendor, allocation of global AVPs should + be encouraged instead. + + AVPs may be allocated following Expert Review (by a Designated + Expert) with Specification Required [RFC5226]. A block allocation + (release of more than three AVPs at a time for a given purpose) + requires IETF Review [RFC5226]. + +11.1.2. AVP Flags + + Section 4.1 describes the existing AVP Flags. The remaining bits can + only be assigned via a Standards Action [RFC5226]. + +11.2. Diameter Header + +11.2.1. Command Codes + + For the Diameter header, the Command Code namespace allocation has + changed. The new allocation rules are as follows: + + The Command Code values 256 - 8,388,607 (0x100 to 0x7fffff) are + for permanent, standard commands, allocated by IETF Review + [RFC5226]. + + The values 8,388,608 - 16,777,213 (0x800000 - 0xfffffd) are + reserved for vendor-specific Command Codes, to be allocated on a + First Come, First Served basis by IANA [RFC5226]. The request to + IANA for a Vendor-Specific Command Code SHOULD include a reference + to a publicly available specification that documents the command + in sufficient detail to aid in interoperability between + independent implementations. If the specification cannot be made + publicly available, the request for a vendor-specific Command Code + MUST include the contact information of persons and/or entities + responsible for authoring and maintaining the command. + + + + + + +Fajardo, et al. Standards Track [Page 136] + +RFC 6733 Diameter Base Protocol October 2012 + + + The values 16,777,214 and 16,777,215 (hexadecimal values 0xfffffe + - 0xffffff) are reserved for experimental commands. As these + codes are only for experimental and testing purposes, no guarantee + is made for interoperability between Diameter peers using + experimental commands. + +11.2.2. Command Flags + + Section 3 describes the existing Command Flags field. The remaining + bits can only be assigned via a Standards Action [RFC5226]. + +11.3. AVP Values + + For AVP values, the Experimental-Result-Code AVP value allocation has + been added; see Section 11.3.1. The old AVP value allocation rule, + IETF Consensus, has been updated to IETF Review as per [RFC5226], and + affected AVPs are listed as reminders. + +11.3.1. Experimental-Result-Code AVP + + Values for this AVP are purely local to the indicated vendor, and no + IANA registry is maintained for them. + +11.3.2. Result-Code AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.3. Accounting-Record-Type AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.4. Termination-Cause AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.5. Redirect-Host-Usage AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.6. Session-Server-Failover AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.7. Session-Binding AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + + + + + +Fajardo, et al. Standards Track [Page 137] + +RFC 6733 Diameter Base Protocol October 2012 + + +11.3.8. Disconnect-Cause AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.9. Auth-Request-Type AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.10. Auth-Session-State AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.11. Re-Auth-Request-Type AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.12. Accounting-Realtime-Required AVP Values + + New values are available for assignment via IETF Review [RFC5226]. + +11.3.13. Inband-Security-Id AVP (code 299) + + The use of this AVP has been deprecated. + +11.4. _diameters Service Name and Port Number Registration + + IANA has registered the "_diameters" service name and assigned port + numbers for TLS/TCP and DTLS/SCTP according to the guidelines given + in [RFC6335]. + + Service Name: _diameters + + Transport Protocols: TCP, SCTP + + Assignee: IESG <[email protected]> + + Contact: IETF Chair <[email protected]> + + Description: Diameter over TLS/TCP and DTLS/SCTP + + Reference: RFC 6733 + + Port Number: 5868, from the User Range + + + + + + + + +Fajardo, et al. Standards Track [Page 138] + +RFC 6733 Diameter Base Protocol October 2012 + + +11.5. SCTP Payload Protocol Identifiers + + Two SCTP payload protocol identifiers have been registered in the + SCTP Payload Protocol Identifiers registry: + + + Value | SCTP Payload Protocol Identifier + -------|----------------------------------- + 46 | Diameter in a SCTP DATA chunk + 47 | Diameter in a DTLS/SCTP DATA chunk + + +11.6. S-NAPTR Parameters + + The following tag has been registered in the S-NAPTR Application + Protocol Tags registry: + + Tag | Protocol + -------------------|--------- + diameter.dtls.sctp | DTLS/SCTP + +12. Diameter Protocol-Related Configurable Parameters + + This section contains the configurable parameters that are found + throughout this document: + + Diameter Peer + + A Diameter entity MAY communicate with peers that are statically + configured. A statically configured Diameter peer would require + that either the IP address or the fully qualified domain name + (FQDN) be supplied, which would then be used to resolve through + DNS. + + Routing Table + + A Diameter proxy server routes messages based on the realm portion + of a Network Access Identifier (NAI). The server MUST have a + table of Realm Names, and the address of the peer to which the + message must be forwarded. The routing table MAY also include a + "default route", which is typically used for all messages that + cannot be locally processed. + + Tc timer + + The Tc timer controls the frequency that transport connection + attempts are done to a peer with whom no active transport + connection exists. The recommended value is 30 seconds. + + + +Fajardo, et al. Standards Track [Page 139] + +RFC 6733 Diameter Base Protocol October 2012 + + +13. Security Considerations + + The Diameter base protocol messages SHOULD be secured by using TLS + [RFC5246] or DTLS/SCTP [RFC6083]. Additional security mechanisms + such as IPsec [RFC4301] MAY also be deployed to secure connections + between peers. However, all Diameter base protocol implementations + MUST support the use of TLS/TCP and DTLS/SCTP, and the Diameter + protocol MUST NOT be used without one of TLS, DTLS, or IPsec. + + If a Diameter connection is to be protected via TLS/TCP and DTLS/SCTP + or IPsec, then TLS/TCP and DTLS/SCTP or IPsec/IKE SHOULD begin prior + to any Diameter message exchange. All security parameters for TLS/ + TCP and DTLS/SCTP or IPsec are configured independent of the Diameter + protocol. All Diameter messages will be sent through the TLS/TCP and + DTLS/SCTP or IPsec connection after a successful setup. + + For TLS/TCP and DTLS/SCTP connections to be established in the open + state, the CER/CEA exchange MUST include an Inband-Security-ID AVP + with a value of TLS/TCP and DTLS/SCTP. The TLS/TCP and DTLS/SCTP + handshake will begin when both ends successfully reach the open + state, after completion of the CER/CEA exchange. If the TLS/TCP and + DTLS/SCTP handshake is successful, all further messages will be sent + via TLS/TCP and DTLS/SCTP. If the handshake fails, both ends MUST + move to the closed state. See Section 13.1 for more details. + +13.1. TLS/TCP and DTLS/SCTP Usage + + Diameter nodes using TLS/TCP and DTLS/SCTP for security MUST mutually + authenticate as part of TLS/TCP and DTLS/SCTP session establishment. + In order to ensure mutual authentication, the Diameter node acting as + the TLS/TCP and DTLS/SCTP server MUST request a certificate from the + Diameter node acting as TLS/TCP and DTLS/SCTP client, and the + Diameter node acting as the TLS/TCP and DTLS/SCTP client MUST be + prepared to supply a certificate on request. + + Diameter nodes MUST be able to negotiate the following TLS/TCP and + DTLS/SCTP cipher suites: + + TLS_RSA_WITH_RC4_128_MD5 + TLS_RSA_WITH_RC4_128_SHA + TLS_RSA_WITH_3DES_EDE_CBC_SHA + + Diameter nodes SHOULD be able to negotiate the following TLS/TCP and + DTLS/SCTP cipher suite: + + TLS_RSA_WITH_AES_128_CBC_SHA + + + + + +Fajardo, et al. Standards Track [Page 140] + +RFC 6733 Diameter Base Protocol October 2012 + + + Note that it is quite possible that support for the + TLS_RSA_WITH_AES_128_CBC_SHA cipher suite will be REQUIRED at some + future date. Diameter nodes MAY negotiate other TLS/TCP and DTLS/ + SCTP cipher suites. + + If public key certificates are used for Diameter security (for + example, with TLS), the value of the expiration times in the routing + and peer tables MUST NOT be greater than the expiry time in the + relevant certificates. + +13.2. Peer-to-Peer Considerations + + As with any peer-to-peer protocol, proper configuration of the trust + model within a Diameter peer is essential to security. When + certificates are used, it is necessary to configure the root + certificate authorities trusted by the Diameter peer. These root CAs + are likely to be unique to Diameter usage and distinct from the root + CAs that might be trusted for other purposes such as Web browsing. + In general, it is expected that those root CAs will be configured so + as to reflect the business relationships between the organization + hosting the Diameter peer and other organizations. As a result, a + Diameter peer will typically not be configured to allow connectivity + with any arbitrary peer. With certificate authentication, Diameter + peers may not be known beforehand and therefore peer discovery may be + required. + +13.3. AVP Considerations + + Diameter AVPs often contain security-sensitive data; for example, + user passwords and location data, network addresses and cryptographic + keys. The following AVPs defined in this document are considered to + be security-sensitive: + + o Acct-Interim-Interval + + o Accounting-Realtime-Required + + o Acct-Multi-Session-Id + + o Accounting-Record-Number + + o Accounting-Record-Type + + o Accounting-Session-Id + + o Accounting-Sub-Session-Id + + o Class + + + +Fajardo, et al. Standards Track [Page 141] + +RFC 6733 Diameter Base Protocol October 2012 + + + o Session-Id + + o Session-Binding + + o Session-Server-Failover + + o User-Name + + Diameter messages containing these or any other AVPs considered to be + security-sensitive MUST only be sent protected via mutually + authenticated TLS or IPsec. In addition, those messages MUST NOT be + sent via intermediate nodes unless there is end-to-end security + between the originator and recipient or the originator has locally + trusted configuration that indicates that end-to-end security is not + needed. For example, end-to-end security may not be required in the + case where an intermediary node is known to be operated as part of + the same administrative domain as the endpoints so that an ability to + successfully compromise the intermediary would imply a high + probability of being able to compromise the endpoints as well. Note + that no end-to-end security mechanism is specified in this document. + +14. References + +14.1. Normative References + + [FLOATPOINT] + Institute of Electrical and Electronics Engineers, "IEEE + Standard for Binary Floating-Point Arithmetic, ANSI/IEEE + Standard 754-1985", August 1985. + + [IANAADFAM] + IANA, "Address Family Numbers", + <http://www.iana.org/assignments/address-family-numbers>. + + [RFC0791] Postel, J., "Internet Protocol", STD 5, RFC 791, + September 1981. + + [RFC0793] Postel, J., "Transmission Control Protocol", STD 7, + RFC 793, September 1981. + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC3492] Costello, A., "Punycode: A Bootstring encoding of Unicode + for Internationalized Domain Names in Applications + (IDNA)", RFC 3492, March 2003. + + + + + +Fajardo, et al. Standards Track [Page 142] + +RFC 6733 Diameter Base Protocol October 2012 + + + [RFC3539] Aboba, B. and J. Wood, "Authentication, Authorization and + Accounting (AAA) Transport Profile", RFC 3539, June 2003. + + [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO + 10646", STD 63, RFC 3629, November 2003. + + [RFC3958] Daigle, L. and A. Newton, "Domain-Based Application + Service Location Using SRV RRs and the Dynamic Delegation + Discovery Service (DDDS)", RFC 3958, January 2005. + + [RFC3986] Berners-Lee, T., Fielding, R., and L. Masinter, "Uniform + Resource Identifier (URI): Generic Syntax", STD 66, + RFC 3986, January 2005. + + [RFC4004] Calhoun, P., Johansson, T., Perkins, C., Hiller, T., and + P. McCann, "Diameter Mobile IPv4 Application", RFC 4004, + August 2005. + + [RFC4005] Calhoun, P., Zorn, G., Spence, D., and D. Mitton, + "Diameter Network Access Server Application", RFC 4005, + August 2005. + + [RFC4006] Hakala, H., Mattila, L., Koskinen, J-P., Stura, M., and J. + Loughney, "Diameter Credit-Control Application", RFC 4006, + August 2005. + + [RFC4086] Eastlake, D., Schiller, J., and S. Crocker, "Randomness + Requirements for Security", BCP 106, RFC 4086, June 2005. + + [RFC4282] Aboba, B., Beadles, M., Arkko, J., and P. Eronen, "The + Network Access Identifier", RFC 4282, December 2005. + + [RFC4291] Hinden, R. and S. Deering, "IP Version 6 Addressing + Architecture", RFC 4291, February 2006. + + [RFC4960] Stewart, R., "Stream Control Transmission Protocol", + RFC 4960, September 2007. + + [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an + IANA Considerations Section in RFCs", BCP 26, RFC 5226, + May 2008. + + [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", STD 68, RFC 5234, January 2008. + + [RFC5246] Dierks, T. and E. Rescorla, "The Transport Layer Security + (TLS) Protocol Version 1.2", RFC 5246, August 2008. + + + + +Fajardo, et al. Standards Track [Page 143] + +RFC 6733 Diameter Base Protocol October 2012 + + + [RFC5280] Cooper, D., Santesson, S., Farrell, S., Boeyen, S., + Housley, R., and W. Polk, "Internet X.509 Public Key + Infrastructure Certificate and Certificate Revocation List + (CRL) Profile", RFC 5280, May 2008. + + [RFC5729] Korhonen, J., Jones, M., Morand, L., and T. Tsou, + "Clarifications on the Routing of Diameter Requests Based + on the Username and the Realm", RFC 5729, December 2009. + + [RFC5890] Klensin, J., "Internationalized Domain Names for + Applications (IDNA): Definitions and Document Framework", + RFC 5890, August 2010. + + [RFC5891] Klensin, J., "Internationalized Domain Names in + Applications (IDNA): Protocol", RFC 5891, August 2010. + + [RFC6083] Tuexen, M., Seggelmann, R., and E. Rescorla, "Datagram + Transport Layer Security (DTLS) for Stream Control + Transmission Protocol (SCTP)", RFC 6083, January 2011. + + [RFC6347] Rescorla, E. and N. Modadugu, "Datagram Transport Layer + Security Version 1.2", RFC 6347, January 2012. + + [RFC6408] Jones, M., Korhonen, J., and L. Morand, "Diameter + Straightforward-Naming Authority Pointer (S-NAPTR) Usage", + RFC 6408, November 2011. + +14.2. Informative References + + [ENTERPRISE] IANA, "SMI Network Management Private Enterprise + Codes", + <http://www.iana.org/assignments/enterprise-numbers>. + + [IANATCV] IANA, "Termination-Cause AVP Values (code 295)", + <http://www.iana.org/assignments/aaa-parameters/ + aaa-parameters.xml#aaa-parameters-16>. + + [RFC1492] Finseth, C., "An Access Control Protocol, Sometimes + Called TACACS", RFC 1492, July 1993. + + [RFC1661] Simpson, W., "The Point-to-Point Protocol (PPP)", + STD 51, RFC 1661, July 1994. + + [RFC2104] Krawczyk, H., Bellare, M., and R. Canetti, "HMAC: + Keyed-Hashing for Message Authentication", RFC 2104, + February 1997. + + + + + +Fajardo, et al. Standards Track [Page 144] + +RFC 6733 Diameter Base Protocol October 2012 + + + [RFC2782] Gulbrandsen, A., Vixie, P., and L. Esibov, "A DNS RR + for specifying the location of services (DNS SRV)", + RFC 2782, February 2000. + + [RFC2865] Rigney, C., Willens, S., Rubens, A., and W. Simpson, + "Remote Authentication Dial In User Service (RADIUS)", + RFC 2865, June 2000. + + [RFC2866] Rigney, C., "RADIUS Accounting", RFC 2866, June 2000. + + [RFC2869] Rigney, C., Willats, W., and P. Calhoun, "RADIUS + Extensions", RFC 2869, June 2000. + + [RFC2881] Mitton, D. and M. Beadles, "Network Access Server + Requirements Next Generation (NASREQNG) NAS Model", + RFC 2881, July 2000. + + [RFC2975] Aboba, B., Arkko, J., and D. Harrington, "Introduction + to Accounting Management", RFC 2975, October 2000. + + [RFC2989] Aboba, B., Calhoun, P., Glass, S., Hiller, T., McCann, + P., Shiino, H., Walsh, P., Zorn, G., Dommety, G., + Perkins, C., Patil, B., Mitton, D., Manning, S., + Beadles, M., Chen, X., Sivalingham, S., Hameed, A., + Munson, M., Jacobs, S., Lim, B., Hirschman, B., Hsu, + R., Koo, H., Lipford, M., Campbell, E., Xu, Y., Baba, + S., and E. Jaques, "Criteria for Evaluating AAA + Protocols for Network Access", RFC 2989, November 2000. + + [RFC3162] Aboba, B., Zorn, G., and D. Mitton, "RADIUS and IPv6", + RFC 3162, August 2001. + + [RFC3748] Aboba, B., Blunk, L., Vollbrecht, J., Carlson, J., and + H. Levkowetz, "Extensible Authentication Protocol + (EAP)", RFC 3748, June 2004. + + [RFC4301] Kent, S. and K. Seo, "Security Architecture for the + Internet Protocol", RFC 4301, December 2005. + + [RFC4690] Klensin, J., Faltstrom, P., Karp, C., and IAB, "Review + and Recommendations for Internationalized Domain Names + (IDNs)", RFC 4690, September 2006. + + [RFC5176] Chiba, M., Dommety, G., Eklund, M., Mitton, D., and B. + Aboba, "Dynamic Authorization Extensions to Remote + Authentication Dial In User Service (RADIUS)", + RFC 5176, January 2008. + + + + +Fajardo, et al. Standards Track [Page 145] + +RFC 6733 Diameter Base Protocol October 2012 + + + [RFC5461] Gont, F., "TCP's Reaction to Soft Errors", RFC 5461, + February 2009. + + [RFC5905] Mills, D., Martin, J., Burbank, J., and W. Kasch, + "Network Time Protocol Version 4: Protocol and + Algorithms Specification", RFC 5905, June 2010. + + [RFC5927] Gont, F., "ICMP Attacks against TCP", RFC 5927, + July 2010. + + [RFC6335] Cotton, M., Eggert, L., Touch, J., Westerlund, M., and + S. Cheshire, "Internet Assigned Numbers Authority + (IANA) Procedures for the Management of the Service + Name and Transport Protocol Port Number Registry", + BCP 165, RFC 6335, August 2011. + + [RFC6737] Kang, J. and G. Zorn, "The Diameter Capabilities Update + Application", RFC 6737, October 2012. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Fajardo, et al. Standards Track [Page 146] + +RFC 6733 Diameter Base Protocol October 2012 + + +Appendix A. Acknowledgements + +A.1. This Document + + The authors would like to thank the following people that have + provided proposals and contributions to this document: + + To Vishnu Ram and Satendra Gera for their contributions on + capabilities updates, predictive loop avoidance, as well as many + other technical proposals. To Tolga Asveren for his insights and + contributions on almost all of the proposed solutions incorporated + into this document. To Timothy Smith for helping on the capabilities + Update and other topics. To Tony Zhang for providing fixes to + loopholes on composing Failed-AVPs as well as many other issues and + topics. To Jan Nordqvist for clearly stating the usage of + Application Ids. To Anders Kristensen for providing needed technical + opinions. To David Frascone for providing invaluable review of the + document. To Mark Jones for providing clarifying text on vendor + command codes and other vendor-specific indicators. To Victor + Pascual and Sebastien Decugis for new text and recommendations on + SCTP/DTLS. To Jouni Korhonen for taking over the editing task and + resolving last bits from versions 27 through 29. + + Special thanks to the Diameter extensibility design team, which + helped resolve the tricky question of mandatory AVPs and ABNF + semantics. The members of this team are as follows: + + Avi Lior, Jari Arkko, Glen Zorn, Lionel Morand, Mark Jones, Tolga + Asveren, Jouni Korhonen, and Glenn McGregor. + + Special thanks also to people who have provided invaluable comments + and inputs especially in resolving controversial issues: + + Glen Zorn, Yoshihiro Ohba, Marco Stura, Stephen Farrel, Pete Resnick, + Peter Saint-Andre, Robert Sparks, Krishna Prasad, Sean Turner, Barry + Leiba, and Pasi Eronen. + + Finally, we would like to thank the original authors of this + document: + + Pat Calhoun, John Loughney, Jari Arkko, Erik Guttman, and Glen Zorn. + + Their invaluable knowledge and experience has given us a robust and + flexible AAA protocol that many people have seen great value in + adopting. We greatly appreciate their support and stewardship for + the continued improvements of Diameter as a protocol. We would also + like to extend our gratitude to folks aside from the authors who have + + + + +Fajardo, et al. Standards Track [Page 147] + +RFC 6733 Diameter Base Protocol October 2012 + + + assisted and contributed to the original version of this document. + Their efforts significantly contributed to the success of Diameter. + +A.2. RFC 3588 + + The authors would like to thank Nenad Trifunovic, Tony Johansson and + Pankaj Patel for their participation in the pre-IETF Document Reading + Party. Allison Mankin, Jonathan Wood, and Bernard Aboba provided + invaluable assistance in working out transport issues and this was + also the case with Steven Bellovin in the security area. + + Paul Funk and David Mitton were instrumental in getting the Peer + State Machine correct, and our deep thanks go to them for their time. + + Text in this document was also provided by Paul Funk, Mark Eklund, + Mark Jones, and Dave Spence. Jacques Caron provided many great + comments as a result of a thorough review of the spec. + + The authors would also like to acknowledge the following people for + their contribution in the development of the Diameter protocol: + + Allan C. Rubens, Haseeb Akhtar, William Bulley, Stephen Farrell, + David Frascone, Daniel C. Fox, Lol Grant, Ignacio Goyret, Nancy + Greene, Peter Heitman, Fredrik Johansson, Mark Jones, Martin Julien, + Bob Kopacz, Paul Krumviede, Fergal Ladley, Ryan Moats, Victor Muslin, + Kenneth Peirce, John Schnizlein, Sumit Vakil, John R. Vollbrecht, and + Jeff Weisberg. + + Finally, Pat Calhoun would like to thank Sun Microsystems since most + of the effort put into this document was done while he was in their + employ. + +Appendix B. S-NAPTR Example + + As an example, consider a client that wishes to resolve aaa: + ex1.example.com. The client performs a NAPTR query for that domain, + and the following NAPTR records are returned: + + ;; order pref flags service regexp replacement + IN NAPTR 50 50 "s" "aaa:diameter.tls.tcp" "" + _diameter._tls.ex1.example.com + IN NAPTR 100 50 "s" "aaa:diameter.tcp" "" + _aaa._tcp.ex1.example.com + IN NAPTR 150 50 "s" "aaa:diameter.sctp" "" + _diameter._sctp.ex1.example.com + + This indicates that the server supports TLS, TCP, and SCTP in that + order. If the client supports TLS, TLS will be used, targeted to a + + + +Fajardo, et al. Standards Track [Page 148] + +RFC 6733 Diameter Base Protocol October 2012 + + + host determined by an SRV lookup of _diameter._tls.ex1.example.com. + That lookup would return: + + ;; Priority Weight Port Target + IN SRV 0 1 5060 server1.ex1.example.com + IN SRV 0 2 5060 server2.ex1.example.com + + As an alternative example, a client that wishes to resolve aaa: + ex2.example.com. The client performs a NAPTR query for that domain, + and the following NAPTR records are returned: + + ;; order pref flags service regexp replacement + IN NAPTR 150 50 "a" "aaa:diameter.tls.tcp" "" + server1.ex2.example.com + IN NAPTR 150 50 "a" "aaa:diameter.tls.tcp" "" + server2.ex2.example.com + + This indicates that the server supports TCP available at the returned + host names. + +Appendix C. Duplicate Detection + + As described in Section 9.4, accounting record duplicate detection is + based on session identifiers. Duplicates can appear for various + reasons: + + o Failover to an alternate server. Where close to real-time + performance is required, failover thresholds need to be kept low. + This may lead to an increased likelihood of duplicates. Failover + can occur at the client or within Diameter agents. + + o Failure of a client or agent after sending a record from non- + volatile memory, but prior to receipt of an application-layer ACK + and deletion of the record to be sent. This will result in + retransmission of the record soon after the client or agent has + rebooted. + + o Duplicates received from RADIUS gateways. Since the + retransmission behavior of RADIUS is not defined within [RFC2865], + the likelihood of duplication will vary according to the + implementation. + + o Implementation problems and misconfiguration. + + The T flag is used as an indication of an application-layer + retransmission event, e.g., due to failover to an alternate server. + It is defined only for request messages sent by Diameter clients or + agents. For instance, after a reboot, a client may not know whether + + + +Fajardo, et al. Standards Track [Page 149] + +RFC 6733 Diameter Base Protocol October 2012 + + + it has already tried to send the accounting records in its non- + volatile memory before the reboot occurred. Diameter servers MAY use + the T flag as an aid when processing requests and detecting duplicate + messages. However, servers that do this MUST ensure that duplicates + are found even when the first transmitted request arrives at the + server after the retransmitted request. It can be used only in cases + where no answer has been received from the server for a request and + the request is sent again, (e.g., due to a failover to an alternate + peer, due to a recovered primary peer or due to a client re-sending a + stored record from non-volatile memory such as after reboot of a + client or agent). + + In some cases, the Diameter accounting server can delay the duplicate + detection and accounting record processing until a post-processing + phase takes place. At that time records are likely to be sorted + according to the included User-Name and duplicate elimination is easy + in this case. In other situations, it may be necessary to perform + real-time duplicate detection, such as when credit limits are imposed + or real-time fraud detection is desired. + + In general, only generation of duplicates due to failover or re- + sending of records in non-volatile storage can be reliably detected + by Diameter clients or agents. In such cases, the Diameter client or + agents can mark the message as a possible duplicate by setting the T + flag. Since the Diameter server is responsible for duplicate + detection, it can choose whether or not to make use of the T flag, in + order to optimize duplicate detection. Since the T flag does not + affect interoperability, and it may not be needed by some servers, + generation of the T flag is REQUIRED for Diameter clients and agents, + but it MAY be implemented by Diameter servers. + + As an example, it can be usually be assumed that duplicates appear + within a time window of longest recorded network partition or device + fault, perhaps a day. So only records within this time window need + to be looked at in the backward direction. Secondly, hashing + techniques or other schemes, such as the use of the T flag in the + received messages, may be used to eliminate the need to do a full + search even in this set except for rare cases. + + The following is an example of how the T flag may be used by the + server to detect duplicate requests. + + A Diameter server MAY check the T flag of the received message to + determine if the record is a possible duplicate. If the T flag is + set in the request message, the server searches for a duplicate + within a configurable duplication time window backward and + forward. This limits database searching to those records where + the T flag is set. In a well-run network, network partitions and + + + +Fajardo, et al. Standards Track [Page 150] + +RFC 6733 Diameter Base Protocol October 2012 + + + device faults will presumably be rare events, so this approach + represents a substantial optimization of the duplicate detection + process. During failover, it is possible for the original record + to be received after the T-flag-marked record, due to differences + in network delays experienced along the path by the original and + duplicate transmissions. The likelihood of this occurring + increases as the failover interval is decreased. In order to be + able to detect duplicates that are out of order, the Diameter + server should use backward and forward time windows when + performing duplicate checking for the T-flag-marked request. For + example, in order to allow time for the original record to exit + the network and be recorded by the accounting server, the Diameter + server can delay processing records with the T flag set until a + time period TIME_WAIT + RECORD_PROCESSING_TIME has elapsed after + the closing of the original transport connection. After this time + period, it may check the T-flag-marked records against the + database with relative assurance that the original records, if + sent, have been received and recorded. + +Appendix D. Internationalized Domain Names + + To be compatible with the existing DNS infrastructure and simplify + host and domain name comparison, Diameter identities (FQDNs) are + represented in ASCII form. This allows the Diameter protocol to fall + in-line with the DNS strategy of being transparent from the effects + of Internationalized Domain Names (IDNs) by following the + recommendations in [RFC4690] and [RFC5890]. Applications that + provide support for IDNs outside of the Diameter protocol but + interacting with it SHOULD use the representation and conversion + framework described in [RFC5890], [RFC5891], and [RFC3492]. +</pre> + +</section> diff --git a/lib/diameter/doc/src/files.mk b/lib/diameter/doc/src/files.mk index 00ced3d91e..510786a7fb 100644 --- a/lib/diameter/doc/src/files.mk +++ b/lib/diameter/doc/src/files.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2012. All Rights Reserved. +# Copyright Ericsson AB 2010-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -38,7 +38,9 @@ XML_REF4_FILES = \ XML_PART_FILES = \ user_man.xml -XML_EXTRA_FILES = +XML_EXTRA_FILES = \ + seealso.ent \ + diameter_soc_rfc6733.xml XML_CHAPTER_FILES = \ diameter_intro.xml \ @@ -50,5 +52,4 @@ XML_CHAPTER_FILES = \ BOOK_FILES = \ book.xml -GIF_FILES = \ - notes.gif +GIF_FILES = diff --git a/lib/diameter/doc/src/notes.gif b/lib/diameter/doc/src/notes.gif Binary files differdeleted file mode 100644 index e000cca26a..0000000000 --- a/lib/diameter/doc/src/notes.gif +++ /dev/null diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index 6fe22492e4..d63e2021c8 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -42,6 +42,93 @@ first.</p> <!-- ===================================================================== --> +<section><title>Diameter 1.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix erroneous watchdog transition from DOWN to INITIAL.</p> + <p> + This transition took place when a peer connection was + reestablished following a failed capabilities exchange. + RFC 3539 requires DOWN to transition into REOPEN.</p> + <p> + Own Id: OTP-10692</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add application_opt() request_errors to make the handling + of incoming requests containing decode errors + configurable.</p> + <p> + The value 'callback' ensures that a handle_request + callback takes place for all such requests, the default + being for diameter to answer 3xxx series errors itself.</p> + <p> + Own Id: OTP-10686</p> + </item> + <item> + <p> + Add transport_opt() length_errors.</p> + <p> + The value determines how messages received over the + transport interface with an incorrect Message Length are + dealt with.</p> + <p> + Own Id: OTP-10687</p> + </item> + <item> + <p> + Add commentary on RFC 6733 to Standards Compliance + chapter of the User's Guide.</p> + <p> + Own Id: OTP-10688</p> + </item> + <item> + <p> + Allow a 5xxx result code in an answer-message on peer + connections using the RFC 6733 common dictionary.</p> + <p> + RFC 6733 allows this while RFC 3588 does not. A + handle_request callback can return {answer_message, + 3000..3999|5000..5999} in the simplest case.</p> + <p> + Own Id: OTP-10759</p> + </item> + <item> + <p> + Add dictionaries for RFC 6733.</p> + <p> + Both the common and accounting dictionaries differ from + their RFC 3588 counterparts, which is reflected in + generated record definitions. Application configuration + on a service or transport determines the dictionary that + will be used on a given peer connection.</p> + <p> + Own Id: OTP-10760</p> + </item> + <item> + <p> + Allow a handle_request callback to control diameter's + setting of Result-Code and Failed-AVP.</p> + <p> + Setting errors = false in a returned #diameter_packet{} + disables the setting.</p> + <p> + Own Id: OTP-10761</p> + </item> + </list> + </section> + +</section> + <section><title>Diameter 1.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/diameter/doc/src/seealso.ent b/lib/diameter/doc/src/seealso.ent index 9945bcadd3..76b9823f79 100644 --- a/lib/diameter/doc/src/seealso.ent +++ b/lib/diameter/doc/src/seealso.ent @@ -4,7 +4,7 @@ %CopyrightBegin% -Copyright Ericsson AB 2012. All Rights Reserved. +Copyright Ericsson AB 2012-2013. All Rights Reserved. The contents of this file are subject to the Erlang Public License, Version 1.1, (the "License"); you may not use this file except in @@ -41,7 +41,8 @@ significant. <!ENTITY mod_origin_state_id '<seealso marker="diameter#origin_state_id-0">diameter:origin_state_id/0</seealso>'> <!ENTITY mod_remove_transport '<seealso marker="diameter#remove_transport-2">diameter:remove_transport/2</seealso>'> <!ENTITY mod_service_info '<seealso marker="diameter#service_info-2">diameter:service_info/2</seealso>'> -<!ENTITY mod_services '<seealso marker="diameter#services-0">diameter:services/0</seealso>'> +<!ENTITY mod_services '<seealso marker="diameter#services-0">diameter:services/0</seealso>'> +<!ENTITY mod_session_id '<seealso marker="diameter#session_id-1">diameter:session_id/1</seealso>'> <!ENTITY mod_start_service '<seealso marker="diameter#start_service-2">diameter:start_service/2</seealso>'> <!ENTITY mod_stop_service '<seealso marker="diameter#stop_service-1">diameter:stop_service/1</seealso>'> <!ENTITY mod_subscribe '<seealso marker="diameter#subscribe-1">diameter:subscribe/1</seealso>'> diff --git a/lib/diameter/include/diameter.hrl b/lib/diameter/include/diameter.hrl index 5ee898c3dd..79c4dce541 100644 --- a/lib/diameter/include/diameter.hrl +++ b/lib/diameter/include/diameter.hrl @@ -143,6 +143,6 @@ init_state, %% option 'state', initial callback state id, %% 32-bit unsigned application identifier = Dict:id() mutable = false, %% boolean(), do traffic callbacks modify state? - options = [{answer_errors, report}]}). %% | callback | discard - + options = [{answer_errors, report}, %% | callback | discard + {request_errors, answer_3xxx}]}). %% | callback | answer -endif. %% -ifdef(diameter_hrl). diff --git a/lib/diameter/src/Makefile b/lib/diameter/src/Makefile index 0e448c8912..df10c33268 100644 --- a/lib/diameter/src/Makefile +++ b/lib/diameter/src/Makefile @@ -109,7 +109,8 @@ ERL_COMPILE_FLAGS += \ +warn_unused_vars \ -pa $(ABS_EBIN) \ -I $(INCDIR) \ - -I gen + -I gen \ + $(STRICT_FLAGS) # -pa is to be able to include_lib from the include directory: the # path must contain the application name. @@ -124,6 +125,13 @@ gen/diameter_gen_%.erl gen/diameter_gen_%.hrl: dict/%.dia opt: $(TARGET_FILES) +# Compile with -Werror during development. Don't do this in the 'opt' +# target so that new warnings don't break the build. It's also +# convenient to have both targets when weeding out warnings isn't the +# priority. (Or when they're intentional, when debugging.) +strict: + $(MAKE) opt STRICT_FLAGS=-Werror + # Build unofficial patches with some degree of traceability. Refuse to # build if there are diffs from HEAD since that defeats the purpose. patch: @@ -251,13 +259,13 @@ release_docs_spec: # Dependencies # ---------------------------------------------------- -gen/diameter_gen_base_accounting.erl gen/diameter_gen_relay.erl \ -gen/diameter_gen_base_accounting.hrl gen/diameter_gen_relay.hrl: \ +gen/diameter_gen_base_accounting.erl gen/diameter_gen_base_accounting.hrl: \ $(EBIN)/diameter_gen_base_rfc3588.$(EMULATOR) gen/diameter_gen_acct_rfc6733.erl gen/diameter_gen_acct_rfc6733.hrl: \ $(EBIN)/diameter_gen_base_rfc6733.$(EMULATOR) +gen/diameter_gen_relay.erl gen/diameter_gen_relay.hrl \ gen/diameter_gen_base_rfc3588.erl gen/diameter_gen_base_rfc3588.hrl \ gen/diameter_gen_base_rfc6733.erl gen/diameter_gen_base_rfc6733.hrl: \ $(COMPILER_MODULES:%=$(EBIN)/%.$(EMULATOR)) @@ -281,7 +289,7 @@ depend.mk: depend.sed $(MODULES:%=%.erl) Makefile .PHONY: debug opt release_docs_spec release_spec .PHONY: $(TARGET_DIRS:%/=%) $(TARGET_DIRS:%/=release_src_%) .PHONY: $(EXAMPLE_DIRS:%/=release_examples_%) -.PHONY: plt dialyze patch +.PHONY: plt dialyze patch strict # Keep intermediate files. .SECONDARY: $(DICT_ERLS) $(DICT_HRLS) gen/$(DICT_YRL:%=%.erl) diff --git a/lib/diameter/src/base/diameter.appup.src b/lib/diameter/src/base/diameter.appup.src index f6d772b534..2ce89579ff 100644 --- a/lib/diameter/src/base/diameter.appup.src +++ b/lib/diameter/src/base/diameter.appup.src @@ -20,14 +20,15 @@ {"%VSN%", [ - {"0.9", [{restart_application, diameter}]}, - {"0.10", [{restart_application, diameter}]}, - {"1.0", [{restart_application, diameter}]}, - {"1.1", [{restart_application, diameter}]}, - {"1.2", [{restart_application, diameter}]}, + {"0.9", [{restart_application, diameter}]}, %% R14B03 + {"0.10", [{restart_application, diameter}]}, %% R14B04 + {"1.0", [{restart_application, diameter}]}, %% R15B + {"1.1", [{restart_application, diameter}]}, %% R15B01 + {"1.2", [{restart_application, diameter}]}, %% R15B02 {"1.2.1", [{restart_application, diameter}]}, - {"1.3", [{restart_application, diameter}]}, - {"1.3.1", [{restart_application, diameter}]} + {"1.3", [{restart_application, diameter}]}, %% R15B03 + {"1.3.1", [{restart_application, diameter}]}, + {"1.4", [{restart_application, diameter}]} %% R16A ], [ {"0.9", [{restart_application, diameter}]}, @@ -37,6 +38,7 @@ {"1.2", [{restart_application, diameter}]}, {"1.2.1", [{restart_application, diameter}]}, {"1.3", [{restart_application, diameter}]}, - {"1.3.1", [{restart_application, diameter}]} + {"1.3.1", [{restart_application, diameter}]}, + {"1.4", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/src/base/diameter.erl b/lib/diameter/src/base/diameter.erl index f563d244f6..c67fba5f89 100644 --- a/lib/diameter/src/base/diameter.erl +++ b/lib/diameter/src/base/diameter.erl @@ -306,7 +306,8 @@ call(SvcName, App, Message) -> | {module, app_module()} | {state, any()} | {call_mutates_state, boolean()} - | {answer_errors, callback|report|discard}. + | {answer_errors, callback|report|discard} + | {request_errors, answer_3xxx|answer|callback}. -type app_alias() :: any(). diff --git a/lib/diameter/src/base/diameter_config.erl b/lib/diameter/src/base/diameter_config.erl index 1486071573..9f73815756 100644 --- a/lib/diameter/src/base/diameter_config.erl +++ b/lib/diameter/src/base/diameter_config.erl @@ -670,15 +670,17 @@ app_acc({application, Opts}, Acc) -> [Dict, Mod] = get_opt([dictionary, module], Opts), Alias = get_opt(alias, Opts, Dict), ModS = get_opt(state, Opts, Alias), - M = get_opt(call_mutates_state, Opts, false), - A = get_opt(answer_errors, Opts, report), + M = get_opt(call_mutates_state, Opts, false, [true]), + A = get_opt(answer_errors, Opts, report, [callback, discard]), + P = get_opt(request_errors, Opts, answer_3xxx, [answer, callback]), [#diameter_app{alias = Alias, dictionary = Dict, id = cb(Dict, id), module = init_mod(Mod), init_state = ModS, - mutable = init_mutable(M), - options = [{answer_errors, init_answers(A)}]} + mutable = M, + options = [{answer_errors, A}, + {request_errors, P}]} | Acc]; app_acc(_, Acc) -> Acc. @@ -707,20 +709,16 @@ init_cb(List) -> V <- [proplists:get_value(F, List, D)]], #diameter_callback{} = list_to_tuple([diameter_callback | Values]). -init_mutable(M) - when M == true; - M == false -> - M; -init_mutable(M) -> - ?THROW({call_mutates_state, M}). - -init_answers(A) - when callback == A; - report == A; - discard == A -> - A; -init_answers(A) -> - ?THROW({answer_errors, A}). +%% Retreive and validate. +get_opt(Key, List, Def, Other) -> + init_opt(Key, get_opt(Key, List, Def), [Def|Other]). + +init_opt(_, V, [V|_]) -> + V; +init_opt(Name, V, [_|Vals]) -> + init_opt(Name, V, Vals); +init_opt(Name, V, []) -> + ?THROW({Name, V}). %% Get a single value at the specified key. get_opt(Keys, List) diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index 0de3825943..f527f7c754 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -20,7 +20,7 @@ %% %% Implements the handling of incoming and outgoing Diameter messages %% except CER/CEA, DWR/DWA and DPR/DPA. That is, the messages that a -%% diameter client sends of receives. +%% diameter client sends and receives. %% -module(diameter_traffic). @@ -38,7 +38,7 @@ failover/1, pending/1]). -%% Other callbacks. +%% towards ?MODULE -export([send/1]). %% send from remote node -include_lib("diameter/include/diameter.hrl"). @@ -187,37 +187,42 @@ recv_request(TPid, Dict0, #recvdata{peerT = PeerT, apps = Apps} = RecvData) -> - recv_request(diameter_service:find_incoming_app(PeerT, TPid, Id, Apps), - TPid, - Pkt, - Dict0, - RecvData). - -%% recv_request/5 - -recv_request({#diameter_app{id = Id, dictionary = Dict} = App, Caps}, - TPid, - Pkt, - Dict0, - RecvData) -> - recv_R(App, + send_A(recv_R(diameter_service:find_incoming_app(PeerT, TPid, Id, Apps), + TPid, + Pkt, + Dict0, + RecvData), TPid, - Caps, Dict0, - RecvData, - diameter_codec:decode(Id, Dict, Pkt)); + RecvData). + +%% recv_R/5 + +recv_R({#diameter_app{id = Id, dictionary = Dict} = App, Caps}, + TPid, + Pkt0, + Dict0, + RecvData) -> + Pkt = errors(Id, diameter_codec:decode(Id, Dict, Pkt0)), + {Caps, Pkt, App, recv_R(App, TPid, Dict0, Caps, RecvData, Pkt)}; %% Note that the decode is different depending on whether or not Id is %% ?APP_ID_RELAY. %% DIAMETER_APPLICATION_UNSUPPORTED 3007 %% A request was sent for an application that is not supported. -recv_request(#diameter_caps{} = Caps, TPid, Pkt, Dict0, _) -> - As = collect_avps(Pkt), - protocol_error(3007, TPid, Caps, Dict0, Pkt#diameter_packet{avps = As}); +recv_R(#diameter_caps{} + = Caps, + _TPid, + #diameter_packet{errors = Es} + = Pkt, + _Dict0, + _RecvData) -> + {Caps, Pkt#diameter_packet{avps = collect_avps(Pkt), + errors = [3007 | Es]}}; -recv_request(false, _, _, _, _) -> %% transport has gone down - ok. +recv_R(false = No, _, _, _, _) -> %% transport has gone down + No. collect_avps(Pkt) -> case diameter_codec:collect_avps(Pkt) of @@ -229,98 +234,51 @@ collect_avps(Pkt) -> %% recv_R/6 -%% Wrong number of bits somewhere in the message: reply. -%% -%% DIAMETER_INVALID_AVP_BITS 3009 -%% A request was received that included an AVP whose flag bits are -%% set to an unrecognized value, or that is inconsistent with the -%% AVP's definition. -%% -recv_R(_App, - TPid, - Caps, - Dict0, - _RecvData, - #diameter_packet{errors = [Bs | _]} = Pkt) - when is_bitstring(Bs) -> - protocol_error(3009, TPid, Caps, Dict0, Pkt); - -%% Either we support this application but don't recognize the command -%% or we're a relay and the command isn't proxiable. -%% -%% DIAMETER_COMMAND_UNSUPPORTED 3001 -%% The Request contained a Command-Code that the receiver did not -%% recognize or support. This MUST be used when a Diameter node -%% receives an experimental command that it does not understand. -%% -recv_R(#diameter_app{id = Id}, - TPid, - Caps, +%% Answer errors ourselves ... +recv_R(#diameter_app{options = [_, {request_errors, E} | _]}, + _TPid, Dict0, + _Caps, _RecvData, - #diameter_packet{header = #diameter_header{is_proxiable = P}, - msg = M} - = Pkt) - when ?APP_ID_RELAY /= Id, undefined == M; - ?APP_ID_RELAY == Id, not P -> - protocol_error(3001, TPid, Caps, Dict0, Pkt); - -%% Error bit was set on a request. -%% -%% DIAMETER_INVALID_HDR_BITS 3008 -%% A request was received whose bits in the Diameter header were -%% either set to an invalid combination, or to a value that is -%% inconsistent with the command code's definition. -%% -recv_R(_App, + #diameter_packet{errors = [RC|_]}) %% a detected 3xxx is hd + when E == answer, (Dict0 /= ?BASE orelse 3 == RC div 1000); + E == answer_3xxx, 3 == RC div 1000 -> + {{answer_message, rc(RC)}, [], []}; + +%% ... or make a handle_request callback. Note that +%% Pkt#diameter_packet.msg = undefined in the 3001 case. +recv_R(App, TPid, + _Dict0, Caps, - Dict0, - _RecvData, - #diameter_packet{header = #diameter_header{is_error = true}} - = Pkt) -> - protocol_error(3008, TPid, Caps, Dict0, Pkt); - -%% A message in a locally supported application or a proxiable message -%% in the relay application. Don't distinguish between the two since -%% each application has its own callback config. That is, the user can -%% easily distinguish between the two cases. -recv_R(App, TPid, Caps, Dict0, RecvData, Pkt) -> - request_cb(App, TPid, Caps, Dict0, RecvData, examine(Pkt)). - -%% Note that there may still be errors but these aren't protocol -%% (3xxx) errors that lead to an answer-message. - -request_cb(App, - TPid, - Caps, - Dict0, - #recvdata{service_name = SvcName} - = RecvData, - Pkt) -> + #recvdata{service_name = SvcName}, + Pkt) -> request_cb(cb(App, handle_request, [Pkt, SvcName, {TPid, Caps}]), App, - TPid, - Caps, - Dict0, - RecvData, [], - Pkt). + []). + +rc({N,_}) -> + N; +rc(N) -> + N. -%% examine/1 +%% errors/1 %% -%% Look for errors in a decoded message. It's odd/unfortunate that -%% 501[15] aren't protocol errors. +%% Look for additional errors in a decoded message, prepending the +%% errors field with the first detected error. It's odd/unfortunate +%% that 501[15] aren't protocol errors. With RFC 3588 this means that +%% a handle_request callback has to formulate the answer. With RFC +%% 6733 it's acceptable for 5xxx to be sent in an answer-message. %% DIAMETER_INVALID_MESSAGE_LENGTH 5015 -%% %% This error is returned when a request is received with an invalid %% message length. -examine(#diameter_packet{header = #diameter_header{length = Len}, - bin = Bin, - errors = Es} - = Pkt) +errors(_, #diameter_packet{header = #diameter_header{length = Len}, + bin = Bin, + errors = Es} + = Pkt) when Len < 20; 0 /= Len rem 4; 8*Len /= bit_size(Bin) -> @@ -330,57 +288,75 @@ examine(#diameter_packet{header = #diameter_header{length = Len}, %% This error is returned when a request was received, whose version %% number is unsupported. -examine(#diameter_packet{header = #diameter_header{version = V}, - errors = Es} - = Pkt) +errors(_, #diameter_packet{header = #diameter_header{version = V}, + errors = Es} + = Pkt) when V /= ?DIAMETER_VERSION -> Pkt#diameter_packet{errors = [5011 | Es]}; -examine(Pkt) -> +%% DIAMETER_INVALID_AVP_BITS 3009 +%% A request was received that included an AVP whose flag bits are +%% set to an unrecognized value, or that is inconsistent with the +%% AVP's definition. + +errors(_, #diameter_packet{errors = [Bs | Es]} = Pkt) + when is_bitstring(Bs) -> + Pkt#diameter_packet{errors = [3009 | Es]}; + +%% DIAMETER_COMMAND_UNSUPPORTED 3001 +%% The Request contained a Command-Code that the receiver did not +%% recognize or support. This MUST be used when a Diameter node +%% receives an experimental command that it does not understand. + +errors(Id, #diameter_packet{header = #diameter_header{is_proxiable = P}, + msg = M, + errors = Es} + = Pkt) + when ?APP_ID_RELAY /= Id, undefined == M; %% don't know the command + ?APP_ID_RELAY == Id, not P -> %% command isn't proxiable + Pkt#diameter_packet{errors = [3001 | Es]}; + +%% DIAMETER_INVALID_HDR_BITS 3008 +%% A request was received whose bits in the Diameter header were +%% either set to an invalid combination, or to a value that is +%% inconsistent with the command code's definition. + +errors(_, #diameter_packet{header = #diameter_header{is_request = true, + is_error = true}, + errors = Es} + = Pkt) -> + Pkt#diameter_packet{errors = [3008 | Es]}; + +%% Green. +errors(_, Pkt) -> Pkt. -%% request_cb/8 +%% request_cb/4 %% A reply may be an answer-message, constructed either here or by %% the handle_request callback. The header from the incoming request %% is passed into the encode so that it can retrieve the relevant %% command code in this case. It will also then ignore Dict and use %% the base encoder. -request_cb({reply, Ans}, - #diameter_app{dictionary = Dict}, - TPid, - _Caps, - Dict0, - _RecvData, - Fs, - Pkt) -> - reply(Ans, dict(Dict, Dict0, Ans), TPid, Fs, Pkt); +request_cb({reply, _Ans} = T, _App, EvalPktFs, EvalFs) -> + {T, EvalPktFs, EvalFs}; %% An 3xxx result code, for which the E-bit is set in the header. -request_cb({protocol_error, RC}, - _App, - TPid, - Caps, - Dict0, - _RecvData, - Fs, - Pkt) - when 3000 =< RC, RC < 4000 -> - protocol_error(RC, TPid, Caps, Dict0, Fs, Pkt); +request_cb({protocol_error, RC}, _App, EvalPktFs, EvalFs) + when 3 == RC div 1000 -> + {{answer_message, RC}, EvalPktFs, EvalFs}; + +request_cb({answer_message, RC} = T, _App, EvalPktFs, EvalFs) + when 3 == RC div 1000; + 5 == RC div 1000 -> + {T, EvalPktFs, EvalFs}; %% RFC 3588 says we must reply 3001 to anything unrecognized or %% unsupported. 'noreply' is undocumented (and inappropriately named) %% backwards compatibility for this, protocol_error the documented %% alternative. -request_cb(noreply, - _App, - TPid, - Caps, - Dict0, - _RecvData, - Fs, - Pkt) -> - protocol_error(3001, TPid, Caps, Dict0, Fs, Pkt); +request_cb(noreply, _App, EvalPktFs, EvalFs) -> + {{answer_message, 3001}, EvalPktFs, EvalFs}; %% Relay a request to another peer. This is equivalent to doing an %% explicit call/4 with the message in question except that (1) a loop @@ -397,29 +373,77 @@ request_cb(noreply, %% want to distinguish between the cases in the callback return value %% then 'resend' is a neutral alternative. %% -request_cb({A, Opts}, - #diameter_app{id = Id} - = App, - TPid, - Caps, - Dict0, - RecvData, - Fs, - Pkt) +request_cb({A, Opts}, #diameter_app{id = Id}, EvalPktFs, EvalFs) when A == relay, Id == ?APP_ID_RELAY; A == proxy, Id /= ?APP_ID_RELAY; A == resend -> - resend(Opts, App, TPid, Caps, Dict0, RecvData, Fs, Pkt); + {{call, Opts}, EvalPktFs, EvalFs}; -request_cb(discard, _, _, _, _, _, _, _) -> - ok; +request_cb(discard = No, _, _, _) -> + No; + +request_cb({eval_packet, RC, F}, App, Fs, EvalFs) -> + request_cb(RC, App, [F|Fs], EvalFs); + +request_cb({eval, RC, F}, App, EvalPktFs, Fs) -> + request_cb(RC, App, EvalPktFs, [F|Fs]); + +request_cb(T, App, _, _) -> + ?ERROR({invalid_return, T, handle_request, App}). + +%% send_A/4 -request_cb({eval_packet, RC, F}, App, TPid, Caps, Dict0, RecvData, Fs, Pkt) -> - request_cb(RC, App, TPid, Caps, Dict0, RecvData, [F|Fs], Pkt); +send_A({Caps, Pkt}, TPid, Dict0, _RecvData) -> %% unsupported application + #diameter_packet{errors = [RC|_]} = Pkt, + send_A(answer_message(RC, Caps, Dict0, Pkt), + TPid, + Dict0, + Pkt, + [], + []); + +send_A({Caps, Pkt, App, {T, EvalPktFs, EvalFs}}, TPid, Dict0, RecvData) -> + send_A(answer(T, Caps, Pkt, App, Dict0, RecvData), + TPid, + Dict0, + Pkt, + EvalPktFs, + EvalFs); + +send_A(_, _, _, _) -> + ok. + +%% send_A/6 + +send_A(T, TPid, Dict0, ReqPkt, EvalPktFs, EvalFs) -> + reply(T, TPid, Dict0, EvalPktFs, ReqPkt), + lists:foreach(fun diameter_lib:eval/1, EvalFs). + +%% answer/6 + +answer({reply, Ans}, _Caps, _Pkt, App, Dict0, _RecvData) -> + {dict(App#diameter_app.dictionary, Dict0, Ans), Ans}; + +answer({call, Opts}, Caps, Pkt, App, Dict0, RecvData) -> + #diameter_caps{origin_host = {OH,_}} + = Caps, + #diameter_packet{avps = Avps} + = Pkt, + {Code, _Flags, Vid} = Dict0:avp_header('Route-Record'), + resend(is_loop(Code, Vid, OH, Dict0, Avps), + Opts, + Caps, + Pkt, + App, + Dict0, + RecvData); -request_cb({eval, RC, F}, App, TPid, Caps, Dict0, RecvData, Fs, Pkt) -> - request_cb(RC, App, TPid, Caps, Dict0, RecvData, Fs, Pkt), - diameter_lib:eval(F). +%% RFC 3588 only allows 3xxx errors in an answer-message. RFC 6733 +%% added the possibility of setting 5xxx. +answer({answer_message, RC} = T, Caps, Pkt, App, Dict0, _RecvData) -> + Dict0 /= ?BASE orelse 3 == RC div 1000 + orelse ?ERROR({invalid_return, T, handle_request, App}), + answer_message(RC, Caps, Dict0, Pkt). %% dict/3 @@ -436,65 +460,31 @@ dict(Dict, Dict0, [Msg]) -> dict(Dict, Dict0, #diameter_packet{msg = Msg}) -> dict(Dict, Dict0, Msg); -dict(_Dict, Dict0, ['answer-message' | _]) -> - Dict0; +dict(Dict, Dict0, Msg) -> + choose(is_answer_message(Msg, Dict0), Dict0, Dict). -dict(Dict, Dict0, Rec) -> +is_answer_message([Name | _], _) -> + Name == 'answer-message'; + +is_answer_message(Rec, Dict) -> try - 'answer-message' = Dict0:rec2msg(element(1,Rec)), - Dict0 + 'answer-message' == Dict:rec2msg(element(1,Rec)) catch - error:_ -> Dict + error:_ -> false end. -%% protocol_error/6 - -protocol_error(RC, TPid, Caps, Dict0, Fs, Pkt) -> - #diameter_caps{origin_host = {OH,_}, - origin_realm = {OR,_}} - = Caps, - #diameter_packet{avps = Avps, errors = Es} - = Pkt, +%% answer_message/4 +answer_message(RC, + #diameter_caps{origin_host = {OH,_}, + origin_realm = {OR,_}}, + Dict0, + #diameter_packet{avps = Avps} + = Pkt) -> ?LOG({error, RC}, Pkt), - reply(answer_message({OH, OR, RC}, Dict0, Avps), - Dict0, - TPid, - Fs, - Pkt#diameter_packet{errors = [RC | Es]}). -%% Note that reply/5 may set the result code once more. It's set in -%% answer_message/3 in case reply/5 doesn't. - -%% protocol_error/5 - -protocol_error(RC, TPid, Caps, Dict0, Pkt) -> - protocol_error(RC, TPid, Caps, Dict0, [], Pkt). + {Dict0, answer_message(OH, OR, RC, Dict0, Avps)}. %% resend/7 -%% -%% Resend a message as a relay or proxy agent. - -resend(Opts, - #diameter_app{} - = App, - TPid, - #diameter_caps{origin_host = {OH,_}} - = Caps, - Dict0, - RecvData, - Fs, - #diameter_packet{avps = Avps} - = Pkt) -> - {Code, _Flags, Vid} = Dict0:avp_header('Route-Record'), - resend(is_loop(Code, Vid, OH, Dict0, Avps), - Opts, - App, - TPid, - Caps, - Dict0, - RecvData, - Fs, - Pkt). %% DIAMETER_LOOP_DETECTED 3005 %% An agent detected a loop while trying to get the message to the @@ -502,8 +492,8 @@ resend(Opts, %% if one is available, but the peer reporting the error has %% identified a configuration problem. -resend(true, _Opts, _App, TPid, Caps, Dict0, _RecvData, Fs, Pkt) -> - protocol_error(3005, TPid, Caps, Dict0, Fs, Pkt); +resend(true, _Opts, Caps, Pkt, _App, Dict0, _RecvData) -> + answer_message(3005, Caps, Dict0, Pkt); %% 6.1.8. Relaying and Proxying Requests %% @@ -513,22 +503,20 @@ resend(true, _Opts, _App, TPid, Caps, Dict0, _RecvData, Fs, Pkt) -> resend(false, Opts, - App, - TPid, #diameter_caps{origin_host = {_,OH}} = Caps, - Dict0, - #recvdata{service_name = SvcName, - sequence = Mask}, - Fs, #diameter_packet{header = Hdr0, avps = Avps} - = Pkt) -> + = Pkt, + App, + Dict0, + #recvdata{service_name = SvcName, + sequence = Mask}) -> Route = #diameter_avp{data = {Dict0, 'Route-Record', OH}}, Seq = diameter_session:sequence(Mask), Hdr = Hdr0#diameter_header{hop_by_hop_id = Seq}, Msg = [Hdr, Route | Avps], - resend(send_request(SvcName, App, Msg, Opts), TPid, Caps, Dict0, Fs, Pkt). + resend(send_request(SvcName, App, Msg, Opts), Caps, Dict0, Pkt). %% The incoming request is relayed with the addition of a %% Route-Record. Note the requirement on the return from call/4 below, %% which places a requirement on the value returned by the @@ -545,28 +533,24 @@ resend(false, %% RFC 6.3 says that a relay agent does not modify Origin-Host but %% says nothing about a proxy. Assume it should behave the same way. -%% resend/6 +%% resend/4 %% %% Relay a reply to a relayed request. %% Answer from the peer: reset the hop by hop identifier and send. resend(#diameter_packet{bin = B} = Pkt, - TPid, _Caps, _Dict0, - Fs, #diameter_packet{header = #diameter_header{hop_by_hop_id = Id}, transport_data = TD}) -> - P = Pkt#diameter_packet{bin = diameter_codec:hop_by_hop_id(Id, B), - transport_data = TD}, - eval_packet(P, Fs), - send(TPid, P); + Pkt#diameter_packet{bin = diameter_codec:hop_by_hop_id(Id, B), + transport_data = TD}; %% TODO: counters %% Or not: DIAMETER_UNABLE_TO_DELIVER. -resend(_, TPid, Caps, Dict0, Fs, Pkt) -> - protocol_error(3002, TPid, Caps, Dict0, Fs, Pkt). +resend(_, Caps, Dict0, Pkt) -> + answer_message(3002, Caps, Dict0, Pkt). %% is_loop/5 %% @@ -590,37 +574,110 @@ is_loop(Code, Vid, OH, Dict0, Avps) -> is_loop(Code, Vid, Dict0:avp(encode, OH, 'Route-Record'), Dict0, Avps). %% reply/5 + +%% Local answer ... +reply({Dict, Ans}, TPid, Dict0, Fs, ReqPkt) -> + reply(Ans, Dict, TPid, Dict0, Fs, ReqPkt); + +%% ... or relayed. +reply(#diameter_packet{} = Pkt, TPid, _Dict0, Fs, _ReqPkt) -> + eval_packet(Pkt, Fs), + send(TPid, Pkt). + +%% reply/6 %% %% Send a locally originating reply. %% Skip the setting of Result-Code and Failed-AVP's below. This is -%% currently undocumented. -reply([Msg], Dict, TPid, Fs, Pkt) +%% undocumented and shouldn't be relied on. +reply([Msg], Dict, TPid, Dict0, Fs, ReqPkt) when is_list(Msg); is_tuple(Msg) -> - reply(Msg, Dict, TPid, Fs, Pkt#diameter_packet{errors = []}); + reply(Msg, Dict, TPid, Dict0, Fs, ReqPkt#diameter_packet{errors = []}); %% No errors or a diameter_header/avp list. -reply(Msg, Dict, TPid, Fs, #diameter_packet{errors = Es} = ReqPkt) - when [] == Es; - is_record(hd(Msg), diameter_header) -> - Pkt = encode(Dict, make_answer_packet(Msg, ReqPkt), Fs), - incr(send, Pkt, Dict, TPid), %% count result codes in sent answers - send(TPid, Pkt); - -%% Or not: set Result-Code and Failed-AVP AVP's. -reply(Msg, Dict, TPid, Fs, #diameter_packet{errors = [H|_] = Es} = Pkt) -> - reply(rc(Msg, rc(H), [A || {_,A} <- Es], Dict), +reply(Msg, Dict, TPid, Dict0, Fs, ReqPkt) -> + Pkt = encode(Dict, reset(make_answer_packet(Msg, ReqPkt), Dict), Fs), + incr(send, Pkt, Dict, TPid, Dict0), %% count outgoing result codes + send(TPid, Pkt). + +%% reset/2 + +%% Header/avps list: send as is. +reset(#diameter_packet{msg = [#diameter_header{} | _]} = Pkt, _) -> + Pkt; + +%% No errors to set or errors explicitly ignored. +reset(#diameter_packet{errors = Es} = Pkt, _) + when Es == []; + Es == false -> + Pkt; + +%% Otherwise possibly set Result-Code and/or Failed-AVP. +reset(#diameter_packet{msg = Msg, errors = Es} = Pkt, Dict) -> + Pkt#diameter_packet{msg = reset(Msg, Dict, Es)}. + +%% reset/3 + +reset(Msg, Dict, Es) + when is_list(Es) -> + {E3, E5, Fs} = partition(Es), + FailedAVP = failed_avp(Msg, lists:reverse(Fs), Dict), + reset(set(Msg, FailedAVP, Dict), Dict, - TPid, - Fs, - Pkt#diameter_packet{errors = []}). + choose(is_answer_message(Msg, Dict), E3, E5)); + +reset(Msg, Dict, N) + when is_integer(N) -> + ResultCode = rc(Msg, {'Result-Code', N}, Dict), + set(Msg, ResultCode, Dict); + +reset(Msg, _, _) -> + Msg. + +partition(Es) -> + lists:foldl(fun pacc/2, {false, false, []}, Es). + +%% Note that the errors list can contain not only integer() and +%% {integer(), #diameter_avp{}} but also #diameter_avp{}. The latter +%% isn't something that's returned by decode but can be set in a reply +%% for encode. + +pacc({RC, #diameter_avp{} = A}, {E3, E5, Acc}) + when is_integer(RC) -> + pacc(RC, {E3, E5, [A|Acc]}); + +pacc(#diameter_avp{} = A, {E3, E5, Acc}) -> + {E3, E5, [A|Acc]}; + +pacc(RC, {false, E5, Acc}) + when 3 == RC div 1000 -> + {RC, E5, Acc}; + +pacc(RC, {E3, false, Acc}) + when 5 == RC div 1000 -> + {E3, RC, Acc}; + +pacc(_, Acc) -> + Acc. eval_packet(Pkt, Fs) -> lists:foreach(fun(F) -> diameter_lib:eval([F,Pkt]) end, Fs). %% make_answer_packet/2 +%% Use decode errors to set Result-Code and/or Failed-AVP unless the +%% the errors field has been explicitly set. Unfortunately, the +%% default value is the empty list rather than 'undefined' so use the +%% atom 'false' for "set nothing". (This is historical and changing +%% the default value would require modules including diameter.hrl to +%% be recompiled.) +make_answer_packet(#diameter_packet{errors = []} + = Pkt, + #diameter_packet{errors = [_|_] = Es} + = ReqPkt) -> + make_answer_packet(Pkt#diameter_packet{errors = Es}, ReqPkt); + %% A reply message clears the R and T flags and retains the P flag. %% The E flag will be set at encode. 6.2 of 3588 requires the same P %% flag on an answer as on the request. A #diameter_packet{} returned @@ -628,6 +685,7 @@ eval_packet(Pkt, Fs) -> %% own header values. make_answer_packet(#diameter_packet{header = Hdr, msg = Msg, + errors = Es, transport_data = TD}, #diameter_packet{header = ReqHdr}) -> Hdr0 = ReqHdr#diameter_header{version = ?DIAMETER_VERSION, @@ -636,6 +694,7 @@ make_answer_packet(#diameter_packet{header = Hdr, is_retransmitted = false}, #diameter_packet{header = fold_record(Hdr0, Hdr), msg = Msg, + errors = Es, transport_data = TD}; %% Binaries and header/avp lists are sent as-is. @@ -652,25 +711,6 @@ make_answer_packet([#diameter_header{} | _] = Msg, make_answer_packet(Msg, #diameter_packet{transport_data = TD} = Pkt) -> make_answer_packet(#diameter_packet{msg = Msg, transport_data = TD}, Pkt). -%% rc/1 - -rc({RC, _}) -> - RC; -rc(RC) -> - RC. - -%% rc/4 - -rc(#diameter_packet{msg = Rec} = Pkt, RC, Failed, DictT) -> - Pkt#diameter_packet{msg = rc(Rec, RC, Failed, DictT)}; - -rc(Rec, RC, Failed, DictT) - when is_integer(RC) -> - set(Rec, - lists:append([rc(Rec, {'Result-Code', RC}, DictT), - failed_avp(Rec, Failed, DictT)]), - DictT). - %% Reply as name and tuple list ... set([_|_] = Ans, Avps, _) -> Ans ++ Avps; %% Values nearer tail take precedence. @@ -796,9 +836,9 @@ fa(Rec, FailedAvp, Dict) -> %% Error-Message AVP is not intended to be useful in real-time, and %% SHOULD NOT be expected to be parsed by network entities. -%% answer_message/3 +%% answer_message/5 -answer_message({OH, OR, RC}, Dict0, Avps) -> +answer_message(OH, OR, RC, Dict0, Avps) -> {Code, _, Vid} = Dict0:avp_header('Session-Id'), ['answer-message', {'Origin-Host', OH}, {'Origin-Realm', OR}, @@ -892,32 +932,48 @@ find(Pred, [H|T]) -> %% incr/4 %% -%% Increment a stats counter for an incoming or outgoing message. +%% Increment a stats counter for result codes in incoming and outgoing +%% answers. %% Outgoing message as binary: don't count. (Sending binaries is only %% partially supported.) -incr(_, #diameter_packet{msg = undefined}, _, _) -> +incr(_, #diameter_packet{msg = undefined}, _, _, _) -> ok; -incr(recv = D, #diameter_packet{header = H, errors = [_|_]}, _, TPid) -> +%% Incoming with decode errors. +incr(recv = D, #diameter_packet{header = H, errors = [_|_]}, _, TPid, _) -> incr(TPid, {diameter_codec:msg_id(H), D, error}); -incr(Dir, Pkt, Dict, TPid) -> +%% Incoming without errors or outgoing. Outgoing with encode errors +%% never gets here since encode fails. +incr(Dir, Pkt, Dict, TPid, Dict0) -> #diameter_packet{header = #diameter_header{is_error = E} = Hdr, msg = Rec} = Pkt, RC = int(get_avp_value(Dict, 'Result-Code', Rec)), - PE = is_protocol_error(RC), - %% Check that the E bit is set only for 3xxx result codes. - (not (E orelse PE)) - orelse (E andalso PE) + %% Exit on an improper Result-Code. + is_result(RC, E, Dict0) orelse x({invalid_error_bit, RC}, answer, [Dir, Pkt]), irc(TPid, Hdr, Dir, rc_counter(Dict, Rec, RC)). +%% No E-bit: can't be 3xxx. +is_result(RC, false, _Dict0) -> + RC < 3000 orelse 4000 =< RC; + +%% E-bit in RFC 3588: only 3xxx. +is_result(RC, true, ?BASE) -> + 3000 =< RC andalso RC < 4000; + +%% E-bit in RFC 6733: 3xxx or 5xxx. +is_result(RC, true, _) -> + 3000 =< RC andalso RC < 4000 + orelse + 5000 =< RC andalso RC < 6000. + irc(_, _, _, undefined) -> false; @@ -929,7 +985,7 @@ irc(TPid, Hdr, Dir, Ctr) -> incr(TPid, Counter) -> diameter_stats:incr(Counter, TPid, 1). -%% error_counter/2 +%% rc_counter/2 %% RFC 3588, 7.6: %% @@ -969,9 +1025,6 @@ int(N) int(_) -> undefined. -is_protocol_error(RC) -> - 3000 =< RC andalso RC < 4000. - -spec x(any(), atom(), list()) -> no_return(). %% Warn and exit request process on errors in an incoming answer. @@ -1121,7 +1174,7 @@ send_R({eval_packet, RC, F}, Pkt, T, Opts, Caller, SvcName, Fs) -> send_R(RC, Pkt, T, Opts, Caller, SvcName, [F|Fs]); send_R(E, _, {_, _, App}, _, _, _, _) -> - ?ERROR({invalid_return, prepare_request, App, E}). + ?ERROR({invalid_return, E, prepare_request, App}). %% make_prepare_packet/2 %% @@ -1268,28 +1321,33 @@ handle_answer(SvcName, App, {error, Req, Reason}) -> handle_error(App, Req, Reason, SvcName); handle_answer(SvcName, - #diameter_app{dictionary = Dict} + #diameter_app{dictionary = Dict, + id = Id} = App, {answer, Req, Dict0, Pkt}) -> Mod = dict(Dict, Dict0, Pkt), - answer(examine(diameter_codec:decode(Mod, Pkt)), - SvcName, - Mod, - App, - Req). + handle_A(errors(Id, diameter_codec:decode(Mod, Pkt)), + SvcName, + Mod, + Dict0, + App, + Req). %% We don't really need to do a full decode if we're a relay and will %% just resend with a new hop by hop identifier, but might a proxy %% want to examine the answer? -answer(Pkt, SvcName, Dict, App, #request{transport = TPid} = Req) -> +handle_A(Pkt, SvcName, Dict, Dict0, App, #request{transport = TPid} = Req) -> try - incr(recv, Pkt, Dict, TPid) + incr(recv, Pkt, Dict, TPid, Dict0) %% count incoming result codes of _ -> answer(Pkt, SvcName, App, Req) catch - exit: {invalid_error_bit, _} = E -> - answer(Pkt#diameter_packet{errors = [E]}, SvcName, App, Req) + exit: {invalid_error_bit, RC} -> + #diameter_packet{errors = Es} + = Pkt, + E = {5004, #diameter_avp{name = 'Result-Code', value = RC}}, + answer(Pkt#diameter_packet{errors = [E|Es]}, SvcName, App, Req) end. answer(Pkt, @@ -1479,7 +1537,7 @@ retransmit({eval_packet, RC, F}, Transport, Req, SvcName, Timeout, Fs) -> retransmit(RC, Transport, Req, SvcName, Timeout, [F|Fs]); retransmit(T, {_, _, App}, _, _, _, _) -> - ?ERROR({invalid_return, prepare_retransmit, App, T}). + ?ERROR({invalid_return, T, prepare_retransmit, App}). resend_request(Pkt0, {TPid, Caps, #diameter_app{dictionary = Dict}}, diff --git a/lib/diameter/src/compiler/diameter_codegen.erl b/lib/diameter/src/compiler/diameter_codegen.erl index 1e31c40afe..80036879ea 100644 --- a/lib/diameter/src/compiler/diameter_codegen.erl +++ b/lib/diameter/src/compiler/diameter_codegen.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -128,8 +128,8 @@ gen(hrl, Spec, Mod, Path) -> gen(erl, Spec, Mod, Path) -> Forms = [{?attribute, module, Mod}, - {?attribute, compile, [{parse_transform, diameter_exprecs}]}, - {?attribute, compile, [{parse_transform, diameter_nowarn}]}, + {?attribute, compile, {parse_transform, diameter_exprecs}}, + {?attribute, compile, nowarn_unused_function}, {?attribute, export, [{name, 0}, {id, 0}, {vendor_id, 0}, diff --git a/lib/diameter/src/compiler/diameter_exprecs.erl b/lib/diameter/src/compiler/diameter_exprecs.erl index cc8458efa1..9e32f53724 100644 --- a/lib/diameter/src/compiler/diameter_exprecs.erl +++ b/lib/diameter/src/compiler/diameter_exprecs.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/diameter/src/compiler/diameter_forms.hrl b/lib/diameter/src/compiler/diameter_forms.hrl index 1a0f3492ea..9b14c1715a 100644 --- a/lib/diameter/src/compiler/diameter_forms.hrl +++ b/lib/diameter/src/compiler/diameter_forms.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/diameter/src/compiler/diameter_nowarn.erl b/lib/diameter/src/compiler/diameter_nowarn.erl deleted file mode 100644 index 6c17af6563..0000000000 --- a/lib/diameter/src/compiler/diameter_nowarn.erl +++ /dev/null @@ -1,41 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2010-2011. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% - -%% -%% A parse transform to work around dialyzer currently not -%% understanding nowarn_unused_function except on individual -%% functions. The include of diameter_gen.hrl by generated dictionary -%% modules contains code that may not be called depending on the -%% dictionary. (The relay dictionary for example.) -%% -%% Even called functions may contain cases that aren't used for a -%% particular dictionary. This also causes dialyzer to complain but -%% there's no way to silence it in this case. -%% - --module(diameter_nowarn). - --export([parse_transform/2]). - -parse_transform(Forms, _Options) -> - [{attribute, ?LINE, compile, {nowarn_unused_function, {F,A}}} - || {function, _, F, A, _} <- Forms] - ++ Forms. -%% Note that dialyzer also doesn't understand {nowarn_unused_function, FAs} -%% with FAs a list of tuples. diff --git a/lib/diameter/src/dict/capup_rfc6737.dia b/lib/diameter/src/dict/capup_rfc6737.dia new file mode 100644 index 0000000000..35d2a9f218 --- /dev/null +++ b/lib/diameter/src/dict/capup_rfc6737.dia @@ -0,0 +1,49 @@ +;; +;; %CopyrightBegin% +;; +;; Copyright Ericsson AB 2013. All Rights Reserved. +;; +;; The contents of this file are subject to the Erlang Public License, +;; Version 1.1, (the "License"); you may not use this file except in +;; compliance with the License. You should have received a copy of the +;; Erlang Public License along with this software. If not, it can be +;; retrieved online at http://www.erlang.org/. +;; +;; Software distributed under the License is distributed on an "AS IS" +;; basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +;; the License for the specific language governing rights and limitations +;; under the License. +;; +;; %CopyrightEnd% +;; + +@id 10 +@name diameter_gen_capup_rfc6737 +@prefix diameter_capup +@vendor 0 IETF + +@inherits diameter_gen_base_rfc6733 + +@messages + + CUR ::= < Diameter Header: 328, REQ > + { Origin-Host } + { Origin-Realm } + 1* { Host-IP-Address } + { Vendor-Id } + { Product-Name } + [ Origin-State-Id ] + * [ Supported-Vendor-Id ] + * [ Auth-Application-Id ] + * [ Acct-Application-Id ] + * [ Vendor-Specific-Application-Id ] + [ Firmware-Revision ] + * [ AVP ] + + + CUA ::= < Diameter Header: 328 > + { Origin-Host } + { Origin-Realm } + { Result-Code } + [ Error-Message ] + * [ AVP ] diff --git a/lib/diameter/src/modules.mk b/lib/diameter/src/modules.mk index 2f3239e1aa..f8d3cf1d6f 100644 --- a/lib/diameter/src/modules.mk +++ b/lib/diameter/src/modules.mk @@ -70,7 +70,6 @@ CT_MODULES = \ base/diameter_info \ compiler/diameter_codegen \ compiler/diameter_exprecs \ - compiler/diameter_nowarn \ compiler/diameter_dict_scanner \ compiler/diameter_dict_util \ compiler/diameter_make diff --git a/lib/diameter/test/.gitignore b/lib/diameter/test/.gitignore index df38dfc5e3..4f19542bbe 100644 --- a/lib/diameter/test/.gitignore +++ b/lib/diameter/test/.gitignore @@ -1,3 +1,4 @@ /log /depend.mk +/coverspec diff --git a/lib/diameter/test/Makefile b/lib/diameter/test/Makefile index aa4b7eaeb1..061f0bcbef 100644 --- a/lib/diameter/test/Makefile +++ b/lib/diameter/test/Makefile @@ -56,7 +56,8 @@ DATA_DIRS = $(sort $(dir $(DATA))) ERL_COMPILE_FLAGS += +warn_export_vars \ +warn_unused_vars \ -I ../include \ - -I ../src/gen + -I ../src/gen \ + $(STRICT_FLAGS) # ---------------------------------------------------- # Targets @@ -64,6 +65,9 @@ ERL_COMPILE_FLAGS += +warn_export_vars \ all debug opt: $(TARGET_FILES) +strict: + $(MAKE) opt STRICT_FLAGS=-Werror + # Require success ... run: $(SUITES) @@ -73,7 +77,7 @@ any: opt clean: rm -f $(TARGET_FILES) - rm -f depend.mk + rm -f depend.mk coverspec realclean: clean rm -rf log @@ -114,7 +118,7 @@ help: @echo " Echo some relevant variables." @echo ======================================== -.PHONY: all any run clean debug docs help info opt realclean +.PHONY: all any run clean debug docs help info opt realclean strict # ---------------------------------------------------- # Special Targets @@ -132,10 +136,21 @@ $(SUITES): log opt | awk '{print} / FAILED /{rc=1} END{exit rc}' rc=0 # Shorter in sed but requires a GNU extension (ie. Q). +cover: log opt coverspec + $(ERL) -noinput \ + -pa $(realpath ../ebin) \ + -sname diameter_cover \ + -s diameter_ct cover \ + -s init stop \ + | awk '{print} / FAILED /{rc=1} END{exit rc}' rc=0 + +coverspec: diameter.cover + sed -f [email protected] $< > $@ + log: mkdir $@ -.PHONY: $(SUITES) +.PHONY: $(SUITES) cover # ---------------------------------------------------- # Release Targets diff --git a/lib/diameter/test/coverspec.sed b/lib/diameter/test/coverspec.sed new file mode 100644 index 0000000000..5e81621593 --- /dev/null +++ b/lib/diameter/test/coverspec.sed @@ -0,0 +1,33 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2013. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +# +# Morph diameter.cover into a legitimate cover spec. All that's being +# retained is the list of excluded modules. This is used by Makefile +# when running cover locally. +# + +/^{incl_app,/{ + i\ +{level, details}.\ +{incl_dirs, ["../ebin"]}. + d +} + +/^{excl_mods,/s@ .*@@ diff --git a/lib/diameter/test/depend.sed b/lib/diameter/test/depend.sed index 7e0d6e40e5..602d1ab497 100644 --- a/lib/diameter/test/depend.sed +++ b/lib/diameter/test/depend.sed @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2011. All Rights Reserved. +# Copyright Ericsson AB 2010-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/diameter/test/diameter_3xxx_SUITE.erl b/lib/diameter/test/diameter_3xxx_SUITE.erl new file mode 100644 index 0000000000..89c78d8b57 --- /dev/null +++ b/lib/diameter/test/diameter_3xxx_SUITE.erl @@ -0,0 +1,509 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Tests of application_opt() request_errors. There's some overlap +%% between this suite and the traffic suite but latter exercises more +%% config. +%% + +-module(diameter_3xxx_SUITE). + +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2]). + +%% testcases +-export([start/1, + send_unknown_application/1, + send_unknown_command/1, + send_ok/1, + send_invalid_avp_bits/1, + send_missing_avp/1, + send_ignore_missing_avp/1, + send_double_error/1, + send_3xxx/1, + send_5xxx/1, + stop/1]). + +%% diameter callbacks +-export([peer_up/3, + peer_down/3, + pick_peer/5, + prepare_request/4, + handle_answer/5, + handle_error/5, + handle_request/3]). + +-include("diameter.hrl"). +-include("diameter_gen_base_rfc6733.hrl"). +%% Use the fact that STR/STA is identical in RFC's 3588 and 6733. + +%% =========================================================================== + +-define(util, diameter_util). +-define(testcase(), proplists:get_value(testcase, get(?MODULE))). +-define(group(Config), begin + put(?MODULE, Config), + ?util:name(proplists:get_value(group, Config)) + end). + +-define(L, atom_to_list). +-define(A, list_to_atom). + +-define(CLIENT, "CLIENT"). +-define(SERVER, "SERVER"). +-define(REALM, "erlang.org"). +-define(HOST(Host, Realm), Host ++ [$.|Realm]). + +-define(ERRORS, [answer, answer_3xxx, callback]). +-define(RFCS, [rfc3588, rfc6733]). +-define(DICT(RFC), ?A("diameter_gen_base_" ++ ?L(RFC))). +-define(DICT, ?DICT(rfc6733)). + +-define(COMMON, ?DIAMETER_APP_ID_COMMON). + +%% Config for diameter:start_service/2. +-define(SERVICE(Name, Errors, RFC), + [{'Origin-Host', Name ++ "." ++ ?REALM}, + {'Origin-Realm', ?REALM}, + {'Host-IP-Address', [{127,0,0,1}]}, + {'Vendor-Id', 12345}, + {'Product-Name', "OTP/diameter"}, + {'Auth-Application-Id', [?COMMON]}, + {application, [{dictionary, ?DICT(RFC)}, + {module, ?MODULE}, + {answer_errors, callback}, + {request_errors, Errors}]}]). + +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). + +%% =========================================================================== + +suite() -> + [{timetrap, {seconds, 60}}]. + +all() -> + [{group, ?util:name([E,D])} || E <- ?ERRORS, D <- ?RFCS]. + +groups() -> + Tc = tc(), + [{?util:name([E,D]), [], [start] ++ Tc ++ [stop]} + || E <- ?ERRORS, D <- ?RFCS]. + +init_per_suite(Config) -> + ok = diameter:start(), + Config. + +end_per_suite(_Config) -> + ok = diameter:stop(). + +init_per_group(Group, Config) -> + [{group, Group} | Config]. + +end_per_group(_, _) -> + ok. + +init_per_testcase(Name, Config) -> + [{testcase, Name} | Config]. + +end_per_testcase(_, _) -> + ok. + +tc() -> + [send_unknown_application, + send_unknown_command, + send_ok, + send_invalid_avp_bits, + send_missing_avp, + send_ignore_missing_avp, + send_double_error, + send_3xxx, + send_5xxx]. + +%% =========================================================================== + +%% start/1 + +start(Config) -> + Group = proplists:get_value(group, Config), + [Errors, RFC] = ?util:name(Group), + ok = diameter:start_service(?SERVER, ?SERVICE(?L(Group), + Errors, + RFC)), + ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT, + callback, + rfc6733)), + LRef = ?util:listen(?SERVER, tcp), + ?util:connect(?CLIENT, tcp, LRef). + +%% stop/1 + +stop(_Config) -> + ok = diameter:remove_transport(?CLIENT, true), + ok = diameter:remove_transport(?SERVER, true), + ok = diameter:stop_service(?SERVER), + ok = diameter:stop_service(?CLIENT). + +%% send_unknown_application/1 +%% +%% Send an unknown application that a callback (which shouldn't take +%% place) fails on. + +%% diameter answers. +send_unknown_application([_,_]) -> + #'diameter_base_answer-message'{'Result-Code' = 3007, + %% UNSUPPORTED_APPLICATION + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +send_unknown_application(Config) -> + send_unknown_application(?group(Config)). + +%% send_unknown_command/1 +%% +%% Send a unknown command that a callback discards. + +%% handle_request discards the request. +send_unknown_command([callback, _]) -> + {error, timeout} = call(); + +%% diameter answers. +send_unknown_command([_,_]) -> + #'diameter_base_answer-message'{'Result-Code' = 3001, + %% UNSUPPORTED_COMMAND + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +send_unknown_command(Config) -> + send_unknown_command(?group(Config)). + +%% send_ok/1 +%% +%% Send a correct STR that a callback answers with 5002. + +%% Callback answers. +send_ok([_,_]) -> + #diameter_base_STA{'Result-Code' = 5002, %% UNKNOWN_SESSION_ID + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +send_ok(Config) -> + send_ok(?group(Config)). + +%% send_invalid_avp_bits/1 +%% +%% Send a request with an incorrect length on the optional +%% Origin-State-Id that a callback ignores. + +%% Callback answers. +send_invalid_avp_bits([callback, _]) -> + #diameter_base_STA{'Result-Code' = 2001, %% SUCCESS + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +%% diameter answers. +send_invalid_avp_bits([_,_]) -> + #'diameter_base_answer-message'{'Result-Code' = 3009, %% INVALID_AVP_BITS + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +send_invalid_avp_bits(Config) -> + send_invalid_avp_bits(?group(Config)). + +%% send_missing_avp/1 +%% +%% Send a request with a missing AVP that a callback answers. + +%% diameter answers. +send_missing_avp([answer, rfc6733]) -> + #'diameter_base_answer-message'{'Result-Code' = 5005, %% MISSING_AVP + 'Failed-AVP' = [_], + 'AVP' = []} + = call(); + +%% Callback answers. +send_missing_avp([_,_]) -> + #diameter_base_STA{'Result-Code' = 5005, %% MISSING_AVP + 'Failed-AVP' = [_], + 'AVP' = []} + = call(); + +send_missing_avp(Config) -> + send_missing_avp(?group(Config)). + +%% send_ignore_missing_avp/1 +%% +%% Send a request with a missing AVP that a callback ignores. + +%% diameter answers. +send_ignore_missing_avp([answer, rfc6733]) -> + #'diameter_base_answer-message'{'Result-Code' = 5005, %% MISSING_AVP + 'Failed-AVP' = [_], + 'AVP' = []} + = call(); + +%% Callback answers, ignores the error +send_ignore_missing_avp([_,_]) -> + #diameter_base_STA{'Result-Code' = 2001, %% SUCCESS + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +send_ignore_missing_avp(Config) -> + send_ignore_missing_avp(?group(Config)). + +%% send_double_error/1 +%% +%% Send a request with both an incorrect length on the optional +%% Origin-State-Id and a missing AVP. + +%% Callback answers with STA. +send_double_error([callback, _]) -> + #diameter_base_STA{'Result-Code' = 5005, %% MISSING_AVP + 'Failed-AVP' = [_], + 'AVP' = []} + = call(); + +%% diameter answers with answer-message. +send_double_error([_,_]) -> + #'diameter_base_answer-message'{'Result-Code' = 3009, %% INVALID_AVP_BITS + 'Failed-AVP' = [_], + 'AVP' = []} + = call(); + +send_double_error(Config) -> + send_double_error(?group(Config)). + +%% send_3xxx/1 +%% +%% Send a request that's answered with a 3xxx result code. + +%% Callback answers. +send_3xxx([_,_]) -> + #'diameter_base_answer-message'{'Result-Code' = 3999, + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +send_3xxx(Config) -> + send_3xxx(?group(Config)). + +%% send_5xxx/1 +%% +%% Send a request that's answered with a 5xxx result code. + +%% Callback answers but fails since 5xxx isn't allowed in an RFC 3588 +%% answer-message. +send_5xxx([_, rfc3588]) -> + {error, timeout} = call(); + +%% Callback answers. +send_5xxx([_,_]) -> + #'diameter_base_answer-message'{'Result-Code' = 5999, + 'Failed-AVP' = [], + 'AVP' = []} + = call(); + +send_5xxx(Config) -> + send_5xxx(?group(Config)). + +%% =========================================================================== + +call() -> + Name = ?testcase(), + diameter:call(?CLIENT, + ?DICT, + #diameter_base_STR + {'Termination-Cause' = ?LOGOUT, + 'Auth-Application-Id' = ?COMMON, + 'Class' = [?L(Name)]}, + [{extra, [Name]}]). + +%% =========================================================================== +%% diameter callbacks + +%% peer_up/3 + +peer_up(_SvcName, _Peer, State) -> + State. + +%% peer_down/3 + +peer_down(_SvcName, _Peer, State) -> + State. + +%% pick_peer/5 + +pick_peer([Peer], _, ?CLIENT, _State, _Name) -> + {ok, Peer}. + +%% prepare_request/4 + +prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, Name) -> + {send, prepare(Pkt, Caps, Name)}. + +prepare(Pkt0, Caps, send_unknown_application) -> + Req = sta(Pkt0, Caps), + #diameter_packet{bin = <<H:8/binary, 0:32, T/binary>>} + = Pkt + = diameter_codec:encode(?DICT, Pkt0#diameter_packet{msg = Req}), + + Pkt#diameter_packet{bin = <<H/binary, 23:32, T/binary>>}; + +prepare(Pkt0, Caps, send_unknown_command) -> + Req = sta(Pkt0, Caps), + #diameter_packet{bin = <<H:5/binary, 275:24, T/binary>>} + = Pkt + = diameter_codec:encode(?DICT, Pkt0#diameter_packet{msg = Req}), + + Pkt#diameter_packet{bin = <<H/binary, 572:24, T/binary>>}; + +prepare(Pkt, Caps, T) + when T == send_ok; + T == send_3xxx; + T == send_5xxx -> + sta(Pkt, Caps); + +prepare(Pkt0, Caps, send_invalid_avp_bits) -> + Req0 = sta(Pkt0, Caps), + %% Append an Origin-State-Id with an incorrect AVP Length in order + %% to force 3009. + Req = Req0#diameter_base_STR{'Origin-State-Id' = [7]}, + #diameter_packet{bin = Bin} + = Pkt + = diameter_codec:encode(?DICT, Pkt0#diameter_packet{msg = Req}), + Offset = size(Bin) - 12 + 5, + <<H:Offset/binary, Len:24, T/binary>> = Bin, + Pkt#diameter_packet{bin = <<H/binary, (Len + 2):24, T/binary>>}; + +prepare(Pkt0, Caps, send_double_error) -> + dehost(prepare(Pkt0, Caps, send_invalid_avp_bits)); + +prepare(Pkt, Caps, T) + when T == send_missing_avp; + T == send_ignore_missing_avp -> + Req = sta(Pkt, Caps), + dehost(diameter_codec:encode(?DICT, Pkt#diameter_packet{msg = Req})). + +sta(Pkt, Caps) -> + #diameter_packet{msg = Req} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, DR}} + = Caps, + Req#diameter_base_STR{'Session-Id' = diameter:session_id(OH), + 'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'Destination-Realm' = DR}. + +%% Strip Origin-Host. +dehost(#diameter_packet{bin = Bin} = Pkt) -> + <<V, Len:24, H:16/binary, T0/binary>> + = Bin, + {SessionId, T1} = split_avp(T0), + {OriginHost, T} = split_avp(T1), + Delta = size(OriginHost), + Pkt#diameter_packet{bin = <<V, (Len - Delta):24, H/binary, + SessionId/binary, + T/binary>>}. + +%% handle_answer/5 + +handle_answer(Pkt, _Req, ?CLIENT, _Peer, _Name) -> + Pkt#diameter_packet.msg. + +%% handle_error/5 + +handle_error(Reason, _Req, ?CLIENT, _Peer, _Name) -> + {error, Reason}. + +split_avp(<<_:5/binary, Len:24, _/binary>> = Bin) -> + L = pad(Len), + <<Avp:L/binary, T/binary>> = Bin, + {Avp, T}. + +pad(N) + when 0 == N rem 4 -> + N; +pad(N) -> + N - (N rem 4) + 4. + +%% handle_request/3 + +handle_request(#diameter_packet{header = #diameter_header{application_id = 0}, + msg = Msg}, + ?SERVER, + {_, Caps}) -> + request(Msg, Caps). + +request(undefined, _) -> %% unknown command + discard; + +request(#diameter_base_STR{'Class' = [Name]} = Req, Caps) -> + request(?A(Name), Req, Caps). + +request(send_ok, Req, Caps) -> + {reply, #diameter_packet{msg = answer(Req, Caps), + errors = [5002]}}; %% UNKNOWN_SESSION_ID + +request(send_3xxx, _Req, _Caps) -> + {answer_message, 3999}; + +request(send_5xxx, _Req, _Caps) -> + {answer_message, 5999}; + +request(send_invalid_avp_bits, Req, Caps) -> + #diameter_base_STR{'Origin-State-Id' = []} + = Req, + %% Default errors field but a non-answer-message and only 3xxx + %% errors detected means diameter sets neither Result-Code nor + %% Failed-AVP. + {reply, #diameter_packet{msg = answer(Req, Caps)}}; + +request(T, Req, Caps) + when T == send_double_error; + T == send_missing_avp -> + {reply, answer(Req, Caps)}; + +request(send_ignore_missing_avp, Req, Caps) -> + {reply, #diameter_packet{msg = answer(Req, Caps), + errors = false}}. %% ignore errors + +answer(Req, Caps) -> + #diameter_base_STR{'Session-Id' = SId} + = Req, + #diameter_caps{origin_host = {OH,_}, + origin_realm = {OR,_}} + = Caps, + #diameter_base_STA{'Session-Id' = SId, + 'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'Result-Code' = 2001}. %% SUCCESS diff --git a/lib/diameter/test/diameter_app_SUITE.erl b/lib/diameter/test/diameter_app_SUITE.erl index 53332af626..209f72adf1 100644 --- a/lib/diameter/test/diameter_app_SUITE.erl +++ b/lib/diameter/test/diameter_app_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -48,7 +48,6 @@ diameter_dict_parser, diameter_dict_util, diameter_exprecs, - diameter_nowarn, diameter_make]). -define(HELP_MODULES, [diameter_dbg, diff --git a/lib/diameter/test/diameter_codec_test.erl b/lib/diameter/test/diameter_codec_test.erl index 18bd2cc190..dc8cbffc83 100644 --- a/lib/diameter/test/diameter_codec_test.erl +++ b/lib/diameter/test/diameter_codec_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/diameter/test/diameter_ct.erl b/lib/diameter/test/diameter_ct.erl index ded50bf6c5..1697287a22 100644 --- a/lib/diameter/test/diameter_ct.erl +++ b/lib/diameter/test/diameter_ct.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -23,16 +23,23 @@ %% Module used to run suites from Makefile. %% --export([run/1]). +-export([run/1, + cover/0]). %% The makefile looks for signs of failure so ignore the ct:run_test/1 %% return value. -run([Suite]) -> +run(Suites) -> + ct_run([{suite, Suites}]). + +cover() -> + ct_run([{spec, "./testspec"}]). + +ct_run(Opts) -> Start = info(), - ct:run_test([{suite, Suite}, - {logdir, "./log"}, - {auto_compile, false}]), + ct:run_test([{logdir, "./log"}, + {auto_compile, false} + | Opts]), info(Start , info()). info() -> diff --git a/lib/diameter/test/diameter_failover_SUITE.erl b/lib/diameter/test/diameter_failover_SUITE.erl index bb820a8bf2..dfd3253827 100644 --- a/lib/diameter/test/diameter_failover_SUITE.erl +++ b/lib/diameter/test/diameter_failover_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -103,9 +103,9 @@ -define(SUCCESS, 2001). %% Value of Termination-Cause determines client/server behaviour. --define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_LOGOUT'). --define(MOVED, ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_USER_MOVED'). --define(TIMEOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_SESSION_TIMEOUT'). +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). +-define(MOVED, ?'DIAMETER_BASE_TERMINATION-CAUSE_USER_MOVED'). +-define(TIMEOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_SESSION_TIMEOUT'). %% =========================================================================== diff --git a/lib/diameter/test/diameter_length_SUITE.erl b/lib/diameter/test/diameter_length_SUITE.erl index 4e413e6a42..ffb19d2288 100644 --- a/lib/diameter/test/diameter_length_SUITE.erl +++ b/lib/diameter/test/diameter_length_SUITE.erl @@ -41,10 +41,10 @@ %% diameter callbacks -export([peer_up/3, peer_down/3, - pick_peer/6, - prepare_request/5, - handle_answer/6, - handle_error/6, + pick_peer/5, + prepare_request/4, + handle_answer/5, + handle_error/5, handle_request/3]). -include("diameter.hrl"). @@ -73,14 +73,14 @@ {answer_errors, callback}]}]). -define(SUCCESS, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_SUCCESS'). + ?'DIAMETER_BASE_RESULT-CODE_SUCCESS'). -define(MISSING_AVP, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_MISSING_AVP'). + ?'DIAMETER_BASE_RESULT-CODE_MISSING_AVP'). -define(INVALID_MESSAGE_LENGTH, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_INVALID_MESSAGE_LENGTH'). + ?'DIAMETER_BASE_RESULT-CODE_INVALID_MESSAGE_LENGTH'). -define(LOGOUT, - ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_LOGOUT'). + ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). -define(GROUPS, [exit, handle, discard]). @@ -196,21 +196,18 @@ send(discard) -> = call(0); send(Config) -> - Group = proplists:get_value(group, Config), - put({?MODULE, group}, Group), - send(Group). + send(proplists:get_value(group, Config)). %% =========================================================================== call(Delta) -> - Group = get({?MODULE, group}), diameter:call(?CLIENT, ?DICT, #diameter_base_STR {'Termination-Cause' = ?LOGOUT, 'Auth-Application-Id' = ?DIAMETER_APP_ID_COMMON, 'Origin-State-Id' = [7]}, - [{extra, [Group, Delta]}]). + [{extra, [Delta]}]). %% =========================================================================== %% diameter callbacks @@ -225,14 +222,14 @@ peer_up(_SvcName, _Peer, State) -> peer_down(_SvcName, _Peer, State) -> State. -%% pick_peer/6 +%% pick_peer/5 -pick_peer([Peer], _, ?CLIENT, _State, _Group, _Delta) -> +pick_peer([Peer], _, ?CLIENT, _State, _Delta) -> {ok, Peer}. -%% prepare_request/5 +%% prepare_request/4 -prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, _Group, Delta) -> +prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, Delta) -> {send, resize(Delta, prepare(Pkt, Caps))}. prepare(#diameter_packet{msg = Req0} = Pkt, Caps) -> @@ -253,14 +250,14 @@ resize(Delta, #diameter_packet{bin = Bin} = Pkt) -> resize(Delta, <<V, Len:24, T/binary>>) -> <<V, (Len + Delta):24, T/binary>>. -%% handle_answer/6 +%% handle_answer/5 -handle_answer(Pkt, _Req, ?CLIENT, _Peer, _Group, _Delta) -> +handle_answer(Pkt, _Req, ?CLIENT, _Peer, _Delta) -> Pkt#diameter_packet.msg. -%% handle_error/6 +%% handle_error/5 -handle_error(Reason, _Req, ?CLIENT, _Peer, _Group, _Delta) -> +handle_error(Reason, _Req, ?CLIENT, _Peer, _Delta) -> {error, Reason}. %% handle_request/3 @@ -280,8 +277,12 @@ handle_request(Pkt, ?SERVER, {_Ref, Caps}) -> answer(Group, #diameter_packet{errors = Es}, Ans) -> answer(Group, Es, Ans); +%% No errors: just answer. answer(_, [], Ans) -> {reply, Ans}; + +%% Otherwise an invalid length should only reach the callback if +%% length_errors = handle. answer(Group, [RC|_], Ans) when RC == ?INVALID_MESSAGE_LENGTH, Group == handle; RC /= ?INVALID_MESSAGE_LENGTH -> diff --git a/lib/diameter/test/diameter_relay_SUITE.erl b/lib/diameter/test/diameter_relay_SUITE.erl index f10d82bdf8..735a908d97 100644 --- a/lib/diameter/test/diameter_relay_SUITE.erl +++ b/lib/diameter/test/diameter_relay_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -107,7 +107,7 @@ -define(LOOP_DETECTED, 3005). -define(UNABLE_TO_DELIVER, 3002). --define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_LOGOUT'). +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). -define(AUTHORIZE_ONLY, ?'DIAMETER_BASE_RE-AUTH-REQUEST-TYPE_AUTHORIZE_ONLY'). %% =========================================================================== diff --git a/lib/diameter/test/diameter_tls_SUITE.erl b/lib/diameter/test/diameter_tls_SUITE.erl index 6cc34b20c5..77194a0f56 100644 --- a/lib/diameter/test/diameter_tls_SUITE.erl +++ b/lib/diameter/test/diameter_tls_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -122,7 +122,7 @@ {capabilities, Caps}]}). -define(SUCCESS, 2001). --define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_LOGOUT'). +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). %% =========================================================================== diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 6727e88b66..781ed234cc 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -178,27 +178,27 @@ diameter_gen_acct_rfc6733]]]). -define(SUCCESS, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_SUCCESS'). + ?'DIAMETER_BASE_RESULT-CODE_SUCCESS'). -define(COMMAND_UNSUPPORTED, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_COMMAND_UNSUPPORTED'). + ?'DIAMETER_BASE_RESULT-CODE_COMMAND_UNSUPPORTED'). -define(TOO_BUSY, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_TOO_BUSY'). + ?'DIAMETER_BASE_RESULT-CODE_TOO_BUSY'). -define(APPLICATION_UNSUPPORTED, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_APPLICATION_UNSUPPORTED'). + ?'DIAMETER_BASE_RESULT-CODE_APPLICATION_UNSUPPORTED'). -define(INVALID_HDR_BITS, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_INVALID_HDR_BITS'). + ?'DIAMETER_BASE_RESULT-CODE_INVALID_HDR_BITS'). -define(INVALID_AVP_BITS, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_INVALID_AVP_BITS'). + ?'DIAMETER_BASE_RESULT-CODE_INVALID_AVP_BITS'). -define(AVP_UNSUPPORTED, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_AVP_UNSUPPORTED'). + ?'DIAMETER_BASE_RESULT-CODE_AVP_UNSUPPORTED'). -define(UNSUPPORTED_VERSION, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_UNSUPPORTED_VERSION'). + ?'DIAMETER_BASE_RESULT-CODE_UNSUPPORTED_VERSION'). -define(REALM_NOT_SERVED, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_REALM_NOT_SERVED'). + ?'DIAMETER_BASE_RESULT-CODE_REALM_NOT_SERVED'). -define(UNABLE_TO_DELIVER, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_UNABLE_TO_DELIVER'). + ?'DIAMETER_BASE_RESULT-CODE_UNABLE_TO_DELIVER'). -define(INVALID_AVP_LENGTH, - ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_INVALID_AVP_LENGTH'). + ?'DIAMETER_BASE_RESULT-CODE_INVALID_AVP_LENGTH'). -define(EVENT_RECORD, ?'DIAMETER_BASE_ACCOUNTING-RECORD-TYPE_EVENT_RECORD'). @@ -208,11 +208,11 @@ ?'DIAMETER_BASE_RE-AUTH-REQUEST-TYPE_AUTHORIZE_AUTHENTICATE'). -define(LOGOUT, - ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_LOGOUT'). + ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). -define(BAD_ANSWER, - ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_BAD_ANSWER'). + ?'DIAMETER_BASE_TERMINATION-CAUSE_BAD_ANSWER'). -define(USER_MOVED, - ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_USER_MOVED'). + ?'DIAMETER_BASE_TERMINATION-CAUSE_USER_MOVED'). %% =========================================================================== @@ -798,7 +798,8 @@ prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, Name, Group) -> prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, send_detach, Group, _) -> {eval_packet, {send, prepare(Pkt, Caps, Group)}, [fun log/2, detach]}. -log(#diameter_packet{} = P, T) -> +log(#diameter_packet{bin = Bin} = P, T) + when is_binary(Bin) -> io:format("~p: ~p~n", [T,P]). %% prepare/4 @@ -980,7 +981,8 @@ answer(T, {Tag, Action, Post}) -> {Tag, answer(T, Action), Post}; answer({A,C}, {reply, Ans}) -> answer(C, {reply, msg(Ans, A, diameter_gen_base_rfc3588)}); -answer(pkt, {reply, Ans}) -> +answer(pkt, {reply, Ans}) + when not is_record(Ans, diameter_packet) -> {reply, #diameter_packet{msg = Ans}}; answer(_, T) -> T. diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk index f575085843..c4a713fb10 100644 --- a/lib/diameter/test/modules.mk +++ b/lib/diameter/test/modules.mk @@ -22,27 +22,28 @@ COVER_SPEC_FILE = diameter.cover MODULES = \ diameter_ct \ - diameter_util \ diameter_enum \ - diameter_compiler_SUITE \ + diameter_util \ + diameter_3xxx_SUITE \ + diameter_app_SUITE \ + diameter_capx_SUITE \ diameter_codec_SUITE \ diameter_codec_test \ - diameter_app_SUITE \ + diameter_compiler_SUITE \ diameter_dict_SUITE \ - diameter_reg_SUITE \ - diameter_sync_SUITE \ - diameter_stats_SUITE \ - diameter_watchdog_SUITE \ + diameter_dpr_SUITE \ + diameter_event_SUITE \ + diameter_failover_SUITE \ diameter_gen_sctp_SUITE \ - diameter_transport_SUITE \ - diameter_capx_SUITE \ - diameter_traffic_SUITE \ + diameter_length_SUITE \ + diameter_reg_SUITE \ diameter_relay_SUITE \ + diameter_stats_SUITE \ + diameter_sync_SUITE \ diameter_tls_SUITE \ - diameter_failover_SUITE \ - diameter_dpr_SUITE \ - diameter_event_SUITE \ - diameter_length_SUITE + diameter_traffic_SUITE \ + diameter_transport_SUITE \ + diameter_watchdog_SUITE HRL_FILES = \ diameter_ct.hrl diff --git a/lib/diameter/test/testspec b/lib/diameter/test/testspec new file mode 100644 index 0000000000..2fd8307281 --- /dev/null +++ b/lib/diameter/test/testspec @@ -0,0 +1,3 @@ + +{suites, ".", all}. +{cover, "./coverspec"}. diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index 74f4c57b70..98e719c50a 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -18,5 +18,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 1.4 +DIAMETER_VSN = 1.4.1 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/doc/src/notes.xml b/lib/edoc/doc/src/notes.xml index 261cae016c..12f93ab400 100644 --- a/lib/edoc/doc/src/notes.xml +++ b/lib/edoc/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2007</year><year>2012</year> + <year>2007</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,6 +31,33 @@ <p>This document describes the changes made to the EDoc application.</p> +<section><title>Edoc 0.7.12</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>EDoc sometimes failed to associate a comment with the + preceding type declaration. This bug has been fixed. + (Thanks to Serge Aleynikov for reporting the bug.) </p> + <p> + Own Id: OTP-10866</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Miscellaneous updates due to Unicode support. </p> + <p> + Own Id: OTP-10820</p> + </item> + </list> + </section> + +</section> + <section><title>Edoc 0.7.11</title> <section><title>Improvements and New Features</title> diff --git a/lib/edoc/src/edoc.erl b/lib/edoc/src/edoc.erl index 599036f380..a87a8471e3 100644 --- a/lib/edoc/src/edoc.erl +++ b/lib/edoc/src/edoc.erl @@ -660,7 +660,7 @@ read_source(Name, Opts0) -> check_forms(Forms, Name), Forms; {error, R} -> - edoc_report:error({"error reading file '~s'.", + edoc_report:error({"error reading file '~ts'.", [edoc_lib:filename(Name)]}), exit({error, R}) end. @@ -677,7 +677,84 @@ read_source_2(Name, Opts) -> Includes = proplists:append_values(includes, Opts) ++ [filename:dirname(Name)], Macros = proplists:append_values(macros, Opts), - epp:parse_file(Name, Includes, Macros). + %% epp:parse_file(Name, Includes, Macros). + parse_file(Name, Includes, Macros). + +%% The code below has been copied from epp.erl. +%% +%% Copy the line of the last token to the last token that will be +%% part of the parse tree. +%% +%% The last line is used in edoc_extract:find_type_docs() to determine +%% if a type declaration is followed by a comment. +%% <example> +%% -type t() :: [ +%% {tag, integer()} +%% ]. +%% %% Protocol options. +%% </example> +%% The line of the dot token will be copied to the integer token. + +parse_file(Name, Includes, Macros) -> + case epp:open(Name, Includes, Macros) of + {ok, Epp} -> + try {ok, parse_file(Epp)} + after _ = epp:close(Epp) + end; + Error -> + Error + end. + +parse_file(Epp) -> + case scan_and_parse(Epp) of + {ok, Form} -> + case Form of + {attribute,La,record,{Record, Fields}} -> + case epp:normalize_typed_record_fields(Fields) of + {typed, NewFields} -> + [{attribute, La, record, {Record, NewFields}}, + {attribute, La, type, + {{record, Record}, Fields, []}} + | parse_file(Epp)]; + not_typed -> + [Form | parse_file(Epp)] + end; + _ -> + [Form | parse_file(Epp)] + end; + {error, E} -> + [{error, E} | parse_file(Epp)]; + {eof, Location} -> + [{eof, Location}] + end. + +scan_and_parse(Epp) -> + case epp:scan_erl_form(Epp) of + {ok, Toks0} -> + Toks = fix_last_line(Toks0), + case erl_parse:parse_form(Toks) of + {ok, Form} -> + {ok, Form}; + Else -> + Else + end; + Else -> + Else + end. + +fix_last_line(Toks0) -> + Toks1 = lists:reverse(Toks0), + {line, LastLine} = erl_scan:token_info(hd(Toks1), line), + fll(Toks1, LastLine, []). + +fll([{Category, Attributes0, Symbol} | L], LastLine, Ts) -> + F = fun(_OldLine) -> LastLine end, + Attributes = erl_scan:set_attribute(line, Attributes0, F), + lists:reverse(L, [{Category, Attributes, Symbol} | Ts]); +fll([T | L], LastLine, Ts) -> + fll(L, LastLine, [T | Ts]); +fll(L, _LastLine, Ts) -> + lists:reverse(L, Ts). check_forms(Fs, Name) -> Fun = fun (F) -> diff --git a/lib/edoc/src/edoc_doclet.erl b/lib/edoc/src/edoc_doclet.erl index a0c1ae1c0f..ce1e94a26a 100644 --- a/lib/edoc/src/edoc_doclet.erl +++ b/lib/edoc/src/edoc_doclet.erl @@ -200,7 +200,7 @@ source({M, P, Name, Path}, Dir, Suffix, Env, Set, Private, Hidden, {Set, Error} end; R -> - report("skipping source file '~s': ~W.", [File, R, 15]), + report("skipping source file '~ts': ~W.", [File, R, 15]), {Set, true} end. @@ -216,14 +216,14 @@ check_name(M, M0, P0, File) -> ok; _ -> if N =/= N0 -> - warning("file '~s' actually contains module '~s'.", + warning("file '~ts' actually contains module '~s'.", [File, M]); true -> ok end end, if P =/= P0 -> - warning("file '~s' belongs to package '~s', not '~s'.", + warning("file '~ts' belongs to package '~s', not '~s'.", [File, P, P0]); true -> ok diff --git a/lib/edoc/src/edoc_lib.erl b/lib/edoc/src/edoc_lib.erl index d9c225e099..276f48453e 100644 --- a/lib/edoc/src/edoc_lib.erl +++ b/lib/edoc/src/edoc_lib.erl @@ -466,20 +466,20 @@ uri_get("file://localhost/" ++ Path) -> uri_get_file(Path); uri_get("file://" ++ Path) -> Msg = io_lib:format("cannot handle 'file:' scheme with " - "nonlocal network-path: 'file://~s'.", + "nonlocal network-path: 'file://~ts'.", [Path]), {error, Msg}; uri_get("file:/" ++ Path) -> uri_get_file(Path); uri_get("file:" ++ Path) -> - Msg = io_lib:format("ignoring malformed URI: 'file:~s'.", [Path]), + Msg = io_lib:format("ignoring malformed URI: 'file:~ts'.", [Path]), {error, Msg}; uri_get("http:" ++ Path) -> uri_get_http("http:" ++ Path); uri_get("ftp:" ++ Path) -> uri_get_ftp("ftp:" ++ Path); uri_get("//" ++ Path) -> - Msg = io_lib:format("cannot access network-path: '//~s'.", [Path]), + Msg = io_lib:format("cannot access network-path: '//~ts'.", [Path]), {error, Msg}; uri_get([C, $:, $/ | _]=Path) when C >= $A, C =< $Z; C >= $a, C =< $z -> uri_get_file(Path); % special case for Windows @@ -490,7 +490,7 @@ uri_get(URI) -> true -> uri_get_file(URI); false -> - Msg = io_lib:format("cannot handle URI: '~s'.", [URI]), + Msg = io_lib:format("cannot handle URI: '~ts'.", [URI]), {error, Msg} end. @@ -555,12 +555,12 @@ uri_get_http_1(Result, URI) -> end. http_errmsg(Reason, URI) -> - io_lib:format("http error: ~s: '~s'", [Reason, URI]). + io_lib:format("http error: ~ts: '~ts'", [Reason, URI]). %% TODO: implement ftp access method uri_get_ftp(URI) -> - Msg = io_lib:format("cannot access ftp scheme yet: '~s'.", [URI]), + Msg = io_lib:format("cannot access ftp scheme yet: '~ts'.", [URI]), {error, Msg}. %% @private @@ -615,7 +615,7 @@ copy_file(From, To) -> {ok, _} -> ok; {error, R} -> R1 = file:format_error(R), - report("error copying '~s' to '~s': ~s.", [From, To, R1]), + report("error copying '~ts' to '~ts': ~ts.", [From, To, R1]), exit(error) end. @@ -631,7 +631,7 @@ list_dir(Dir, Error) -> fun (S, As) -> warning(S, As), [] end end, R1 = file:format_error(R), - F("could not read directory '~s': ~s.", [filename(Dir), R1]) + F("could not read directory '~ts': ~ts.", [filename(Dir), R1]) end. %% @private @@ -667,7 +667,7 @@ simplify_path(P) -> %% ok -> ok; %% {error, R} -> %% R1 = file:format_error(R), -%% report("cannot create directory '~s': ~s.", [Dir, R1]), +%% report("cannot create directory '~ts': ~ts.", [Dir, R1]), %% exit(error) %% end. @@ -707,7 +707,7 @@ write_file(Text, Dir, Name, Package, Options) -> ok = file:close(FD); {error, R} -> R1 = file:format_error(R), - report("could not write file '~s': ~s.", [File, R1]), + report("could not write file '~ts': ~ts.", [File, R1]), exit(error) end. @@ -761,7 +761,7 @@ read_info_file(Dir) -> parse_info_file(Text, File); {error, R} -> R1 = file:format_error(R), - warning("could not read '~s': ~s.", [File, R1]), + warning("could not read '~ts': ~ts.", [File, R1]), {?NO_APP, [], []} end; false -> @@ -776,7 +776,7 @@ uri_get_info_file(Base) -> {ok, Text} -> parse_info_file(Text, URI); {error, Msg} -> - warning("could not read '~s': ~s.", [URI, Msg]), + warning("could not read '~ts': ~ts.", [URI, Msg]), {?NO_APP, [], []} end. @@ -785,10 +785,10 @@ parse_info_file(Text, Name) -> {ok, Vs} -> info_file_data(Vs); {error, eof} -> - warning("unexpected end of file in '~s'.", [Name]), + warning("unexpected end of file in '~ts'.", [Name]), {?NO_APP, [], []}; {error, {_Line,Module,R}} -> - warning("~s: ~s.", [Module:format_error(R), Name]), + warning("~ts: ~ts.", [Module:format_error(R), Name]), {?NO_APP, [], []} end. @@ -1033,7 +1033,7 @@ run_plugin(Name, Key, Default, Fun, Opts) when is_atom(Name) -> {ok, Value} -> Value; R -> - report("error in ~s '~w': ~W.", [Name, Module, R, 20]), + report("error in ~ts '~w': ~W.", [Name, Module, R, 20]), exit(error) end. diff --git a/lib/edoc/src/edoc_macros.erl b/lib/edoc/src/edoc_macros.erl index 08686c4fb5..8efbfd00c7 100644 --- a/lib/edoc/src/edoc_macros.erl +++ b/lib/edoc/src/edoc_macros.erl @@ -88,7 +88,7 @@ link_macro(S, Line, Env) -> true -> " target=\"_top\""; % note the initial space false -> "" end, - lists:flatten(io_lib:fwrite("<a href=\"~s\"~s>~ts</a>", + lists:flatten(io_lib:fwrite("<a href=\"~ts\"~ts>~ts</a>", [URI, Target, Txt])). section_macro(S, _Line, _Env) -> @@ -102,7 +102,7 @@ type_macro(S, Line, Env) -> Def = edoc_parser:parse_typedef(S1, Line), {#t_typedef{type = T}, _} = Def, Txt = edoc_layout:type(edoc_data:type(T, Env)), - lists:flatten(io_lib:fwrite("<code>~s</code>", [Txt])). + lists:flatten(io_lib:fwrite("<code>~ts</code>", [Txt])). %% Expand inline macros in tag content. diff --git a/lib/edoc/src/edoc_parser.yrl b/lib/edoc/src/edoc_parser.yrl index cf1a2d6b11..a20f152f34 100644 --- a/lib/edoc/src/edoc_parser.yrl +++ b/lib/edoc/src/edoc_parser.yrl @@ -462,4 +462,4 @@ throw_error({parse_throws, E}, L) -> throw_error(parse_param, L) -> throw({error, L, "missing parameter name"}); throw_error({Where, E}, L) when is_list(Where) -> - throw({error,L,{"unknown error parsing ~s: ~P.",[Where,E,15]}}). + throw({error,L,{"unknown error parsing ~ts: ~P.",[Where,E,15]}}). diff --git a/lib/edoc/src/edoc_report.erl b/lib/edoc/src/edoc_report.erl index 9bec08ab97..dc6320df6d 100644 --- a/lib/edoc/src/edoc_report.erl +++ b/lib/edoc/src/edoc_report.erl @@ -83,13 +83,13 @@ report(L, Where, S, Vs) -> io:nl(). where({File, module}) -> - io_lib:fwrite("~s, in module header: ", [File]); + io_lib:fwrite("~ts, in module header: ", [File]); where({File, footer}) -> - io_lib:fwrite("~s, in module footer: ", [File]); + io_lib:fwrite("~ts, in module footer: ", [File]); where({File, header}) -> - io_lib:fwrite("~s, in header file: ", [File]); + io_lib:fwrite("~ts, in header file: ", [File]); where({File, {F, A}}) -> - io_lib:fwrite("~s, function ~s/~w: ", [File, F, A]); + io_lib:fwrite("~ts, function ~s/~w: ", [File, F, A]); where([]) -> io_lib:fwrite("~s: ", [?APPLICATION]); where(File) when is_list(File) -> diff --git a/lib/edoc/src/edoc_run.erl b/lib/edoc/src/edoc_run.erl index 48b6137ac1..b5a1ef713d 100644 --- a/lib/edoc/src/edoc_run.erl +++ b/lib/edoc/src/edoc_run.erl @@ -162,7 +162,7 @@ file(Args) -> -spec invalid_args(string(), list()) -> no_return(). invalid_args(Where, Args) -> - report("invalid arguments to ~s: ~w.", [Where, Args]), + report("invalid arguments to ~ts: ~w.", [Where, Args]), shutdown_error(). run(F) -> @@ -213,13 +213,13 @@ parse_arg(A) -> {ok, Expr} -> case catch erl_parse:normalise(Expr) of {'EXIT', _} -> - report("bad argument: '~s':", [A]), + report("bad argument: '~ts':", [A]), exit(error); Term -> Term end; {error, _, D} -> - report("error parsing argument '~s'", [A]), + report("error parsing argument '~ts'", [A]), error(D), exit(error) end. diff --git a/lib/edoc/src/edoc_tags.erl b/lib/edoc/src/edoc_tags.erl index 2d986988c2..eb41f1922a 100644 --- a/lib/edoc/src/edoc_tags.erl +++ b/lib/edoc/src/edoc_tags.erl @@ -391,10 +391,10 @@ parse_header(Data, Line, Env, Where) when is_list(Where) -> -spec throw_error(line(), err()) -> no_return(). throw_error(L, {read_file, File, R}) -> - throw_error(L, {"error reading file '~s': ~w", + throw_error(L, {"error reading file '~ts': ~w", [edoc_lib:filename(File), R]}); throw_error(L, {file_not_found, F}) -> - throw_error(L, {"file not found: ~s", [F]}); + throw_error(L, {"file not found: ~ts", [F]}); throw_error(L, file_not_string) -> throw_error(L, "expected file name as a string"); throw_error(L, D) -> diff --git a/lib/edoc/src/edoc_wiki.erl b/lib/edoc/src/edoc_wiki.erl index cc0529d2a9..5d0d78bf3c 100644 --- a/lib/edoc/src/edoc_wiki.erl +++ b/lib/edoc/src/edoc_wiki.erl @@ -295,7 +295,7 @@ expand_uri([], _, L, _Ss, Us, _As) -> expand_uri_error(Us, L) -> {Ps, _} = edoc_lib:split_at(lists:reverse(Us), $:), - throw_error(L, {"reference '[~s:...' ended unexpectedly", [Ps]}). + throw_error(L, {"reference '[~ts:...' ended unexpectedly", [Ps]}). push_uri(Us, Ss, As) -> diff --git a/lib/edoc/vsn.mk b/lib/edoc/vsn.mk index cfde80f0fa..af2aa2203f 100644 --- a/lib/edoc/vsn.mk +++ b/lib/edoc/vsn.mk @@ -1 +1 @@ -EDOC_VSN = 0.7.11 +EDOC_VSN = 0.7.12 diff --git a/lib/eldap/doc/src/notes.xml b/lib/eldap/doc/src/notes.xml index 6dd3973c81..2cdba83bcd 100644 --- a/lib/eldap/doc/src/notes.xml +++ b/lib/eldap/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2012</year> + <year>2012</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/erl_docgen/doc/src/notes.xml b/lib/erl_docgen/doc/src/notes.xml index 5b255efe31..abd3a663bd 100644 --- a/lib/erl_docgen/doc/src/notes.xml +++ b/lib/erl_docgen/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index dfe181bd1d..6c340378d4 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -264,7 +264,9 @@ typedef enum { The <c>p</c> parameter is the name of the atom with character encoding <c><seealso marker="#erlang_char_encoding">from_enc</seealso></c> (ascii, latin1 or utf8). The name must either be zero-terminated or a function variant with a <c>len</c> - parameter must be used.</p> + parameter must be used. If <c>to_enc</c> is set to the bitwise-or'd combination + <c>(ERLANG_LATIN1|ERLANG_UTF8)</c>, utf8 encoding is only used if the atom string + can not be represented in latin1 encoding.</p> <p>The encoding will fail if <c>p</c> is not a valid string in encoding <c>from_enc</c>, if the string is too long or if it can not be represented with character encoding <c>to_enc</c>.</p> <p>These functions were introduced in R16 release of Erlang/OTP as part of a first step diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index 7cf3d46d88..ae25f0c9ab 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,24 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 3.7.11</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Revert the structs <c>erlang_pid</c>, <c>erlang_port</c> + and <c>erlang_ref</c> as they were before R16A (without + <c>node_org_enc</c>) in order to be backward compatible + with user code that accesses the fields of these structs.</p> + <p> + Own Id: OTP-10885 Aux Id: seq12256 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.7.10</title> <section><title>Improvements and New Features</title> diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index 66dc64a69d..f51f377b9c 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -199,7 +199,6 @@ typedef enum { /* a pid */ typedef struct { char node[MAXATOMLEN_UTF8]; - erlang_char_encoding node_org_enc; unsigned int num; unsigned int serial; unsigned int creation; @@ -208,7 +207,6 @@ typedef struct { /* a port */ typedef struct { char node[MAXATOMLEN_UTF8]; - erlang_char_encoding node_org_enc; unsigned int id; unsigned int creation; } erlang_port; @@ -216,7 +214,6 @@ typedef struct { /* a ref */ typedef struct { char node[MAXATOMLEN_UTF8]; - erlang_char_encoding node_org_enc; int len; unsigned int n[3]; unsigned int creation; diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index c1361e169e..3ab86bb340 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -459,7 +459,6 @@ int ei_connect_xinit(ei_cnode* ec, const char *thishostname, /* memmove(&ec->this_ipaddr, thisipaddr, sizeof(ec->this_ipaddr)); */ strcpy(ec->self.node,thisnodename); - ec->self.node_org_enc = ERLANG_LATIN1; ec->self.num = 0; ec->self.serial = 0; ec->self.creation = creation; diff --git a/lib/erl_interface/src/decode/decode_pid.c b/lib/erl_interface/src/decode/decode_pid.c index d429fb2fd8..cd5ae2ab20 100644 --- a/lib/erl_interface/src/decode/decode_pid.c +++ b/lib/erl_interface/src/decode/decode_pid.c @@ -30,7 +30,7 @@ int ei_decode_pid(const char *buf, int *index, erlang_pid *p) if (get8(s) != ERL_PID_EXT) return -1; if (p) { - if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1; + if (get_atom(&s, p->node, NULL) < 0) return -1; p->num = get32be(s) & 0x7fff; /* 15 bits */ p->serial = get32be(s) & 0x1fff; /* 13 bits */ p->creation = get8(s) & 0x03; /* 2 bits */ diff --git a/lib/erl_interface/src/decode/decode_port.c b/lib/erl_interface/src/decode/decode_port.c index 7a691f0be6..8fbdc5f3d3 100644 --- a/lib/erl_interface/src/decode/decode_port.c +++ b/lib/erl_interface/src/decode/decode_port.c @@ -29,7 +29,7 @@ int ei_decode_port(const char *buf, int *index, erlang_port *p) if (get8(s) != ERL_PORT_EXT) return -1; if (p) { - if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1; + if (get_atom(&s, p->node, NULL) < 0) return -1; p->id = get32be(s) & 0x0fffffff /* 28 bits */; p->creation = get8(s) & 0x03; } diff --git a/lib/erl_interface/src/decode/decode_ref.c b/lib/erl_interface/src/decode/decode_ref.c index 01e3061cb4..78db118172 100644 --- a/lib/erl_interface/src/decode/decode_ref.c +++ b/lib/erl_interface/src/decode/decode_ref.c @@ -31,7 +31,7 @@ int ei_decode_ref(const char *buf, int *index, erlang_ref *p) switch (get8(s)) { case ERL_REFERENCE_EXT: if (p) { - if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1; + if (get_atom(&s, p->node, NULL) < 0) return -1; p->n[0] = get32be(s); p->len = 1; p->creation = get8(s) & 0x03; @@ -52,7 +52,7 @@ int ei_decode_ref(const char *buf, int *index, erlang_ref *p) if (p) { p->len = count; - if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1; + if (get_atom(&s, p->node, NULL) < 0) return -1; p->creation = get8(s) & 0x03; } else { diff --git a/lib/erl_interface/src/decode/decode_skip.c b/lib/erl_interface/src/decode/decode_skip.c index e2bfe1f802..553266471c 100644 --- a/lib/erl_interface/src/decode/decode_skip.c +++ b/lib/erl_interface/src/decode/decode_skip.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/erl_interface/src/decode/decode_trace.c b/lib/erl_interface/src/decode/decode_trace.c index 88fb3451ec..d192c7f8b1 100644 --- a/lib/erl_interface/src/decode/decode_trace.c +++ b/lib/erl_interface/src/decode/decode_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/erl_interface/src/encode/encode_atom.c b/lib/erl_interface/src/encode/encode_atom.c index df4b0af5db..46d34c3bf0 100644 --- a/lib/erl_interface/src/encode/encode_atom.c +++ b/lib/erl_interface/src/encode/encode_atom.c @@ -25,7 +25,7 @@ static int verify_ascii_atom(const char* src, int slen); static int verify_utf8_atom(const char* src, int slen); - +static int is_latin1_as_utf8(const char *p, int len); int ei_encode_atom(char *buf, int *index, const char *p) { @@ -63,6 +63,14 @@ int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len, return -1; } + if (to_enc == (ERLANG_LATIN1 | ERLANG_UTF8)) { + if (from_enc == ERLANG_UTF8) { + to_enc = is_latin1_as_utf8(p, len) ? ERLANG_LATIN1 : ERLANG_UTF8; + } + else { + to_enc = from_enc; + } + } switch(to_enc) { case ERLANG_LATIN1: if (buf) { @@ -148,7 +156,7 @@ ei_internal_put_atom(char** bufp, const char* p, int slen, } -int verify_ascii_atom(const char* src, int slen) +static int verify_ascii_atom(const char* src, int slen) { while (slen > 0) { if ((src[0] & 0x80) != 0) return -1; @@ -158,7 +166,7 @@ int verify_ascii_atom(const char* src, int slen) return 0; } -int verify_utf8_atom(const char* src, int slen) +static int verify_utf8_atom(const char* src, int slen) { int num_chars = 0; @@ -188,3 +196,13 @@ int verify_utf8_atom(const char* src, int slen) return 0; } +/* Only latin1 code points in utf8 string? + */ +static int is_latin1_as_utf8(const char *p, int len) +{ + int i; + for (i=0; i<len; i++) { + if ((unsigned char)p[i] > 0xC3) return 0; + } + return 1; +} diff --git a/lib/erl_interface/src/encode/encode_pid.c b/lib/erl_interface/src/encode/encode_pid.c index 903c9cce00..86d0f393e5 100644 --- a/lib/erl_interface/src/encode/encode_pid.c +++ b/lib/erl_interface/src/encode/encode_pid.c @@ -26,7 +26,8 @@ int ei_encode_pid(char *buf, int *index, const erlang_pid *p) char *s = buf + *index; ++(*index); /* skip ERL_PID_EXT */ - if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), ERLANG_UTF8, p->node_org_enc) < 0) + if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), + ERLANG_UTF8, ERLANG_LATIN1|ERLANG_UTF8) < 0) return -1; if (buf) { diff --git a/lib/erl_interface/src/encode/encode_port.c b/lib/erl_interface/src/encode/encode_port.c index c729aeb4eb..a206de56c7 100644 --- a/lib/erl_interface/src/encode/encode_port.c +++ b/lib/erl_interface/src/encode/encode_port.c @@ -27,7 +27,7 @@ int ei_encode_port(char *buf, int *index, const erlang_port *p) ++(*index); /* skip ERL_PORT_EXT */ if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), ERLANG_UTF8, - p->node_org_enc) < 0) { + ERLANG_LATIN1|ERLANG_UTF8) < 0) { return -1; } if (buf) { diff --git a/lib/erl_interface/src/encode/encode_ref.c b/lib/erl_interface/src/encode/encode_ref.c index 3511366bef..9855231848 100644 --- a/lib/erl_interface/src/encode/encode_ref.c +++ b/lib/erl_interface/src/encode/encode_ref.c @@ -28,7 +28,7 @@ int ei_encode_ref(char *buf, int *index, const erlang_ref *p) (*index) += 1 + 2; /* skip to node atom */ if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), ERLANG_UTF8, - p->node_org_enc) < 0) { + ERLANG_LATIN1|ERLANG_UTF8) < 0) { return -1; } diff --git a/lib/erl_interface/src/legacy/erl_connect.c b/lib/erl_interface/src/legacy/erl_connect.c index eca16497dc..ae0265a388 100644 --- a/lib/erl_interface/src/legacy/erl_connect.c +++ b/lib/erl_interface/src/legacy/erl_connect.c @@ -250,11 +250,9 @@ int erl_send(int fd, ETERM *to ,ETERM *msg) if (to->uval.pidval.node.latin1) { strcpy(topid.node, to->uval.pidval.node.latin1); - topid.node_org_enc = ERLANG_LATIN1; } else { strcpy(topid.node, to->uval.pidval.node.utf8); - topid.node_org_enc = ERLANG_UTF8; } topid.num = ERL_PID_NUMBER(to); topid.serial = ERL_PID_SERIAL(to); diff --git a/lib/erl_interface/src/misc/ei_decode_term.c b/lib/erl_interface/src/misc/ei_decode_term.c index 1423ec7ed7..ce5ae5b19d 100644 --- a/lib/erl_interface/src/misc/ei_decode_term.c +++ b/lib/erl_interface/src/misc/ei_decode_term.c @@ -54,7 +54,7 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term) return ei_decode_atom(buf, index, term->value.atom_name); case ERL_REFERENCE_EXT: /* first the nodename */ - if (get_atom(&s, term->value.ref.node, &term->value.ref.node_org_enc) < 0) return -1; + if (get_atom(&s, term->value.ref.node, NULL) < 0) return -1; /* now the numbers: num (4), creation (1) */ term->value.ref.n[0] = get32be(s); term->value.ref.len = 1; @@ -64,7 +64,7 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term) /* first the integer count */ term->value.ref.len = get16be(s); /* then the nodename */ - if (get_atom(&s, term->value.ref.node, &term->value.ref.node_org_enc) < 0) return -1; + if (get_atom(&s, term->value.ref.node, NULL) < 0) return -1; /* creation */ term->value.ref.creation = get8(s) & 0x03; /* finally the id integers */ @@ -76,12 +76,12 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term) } break; case ERL_PORT_EXT: - if (get_atom(&s, term->value.port.node, &term->value.port.node_org_enc) < 0) return -1; + if (get_atom(&s, term->value.port.node, NULL) < 0) return -1; term->value.port.id = get32be(s) & 0x0fffffff; /* 28 bits */; term->value.port.creation = get8(s) & 0x03; break; case ERL_PID_EXT: - if (get_atom(&s, term->value.pid.node, &term->value.port.node_org_enc) < 0) return -1; + if (get_atom(&s, term->value.pid.node, NULL) < 0) return -1; /* now the numbers: num (4), serial (4), creation (1) */ term->value.pid.num = get32be(s) & 0x7fff; /* 15 bits */ term->value.pid.serial = get32be(s) & 0x1fff; /* 13 bits */ diff --git a/lib/erl_interface/test/all_SUITE_data/init_tc.erl b/lib/erl_interface/test/all_SUITE_data/init_tc.erl index 0f2ac215f9..5245173260 100644 --- a/lib/erl_interface/test/all_SUITE_data/init_tc.erl +++ b/lib/erl_interface/test/all_SUITE_data/init_tc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/erl_interface/test/all_SUITE_data/runner.c b/lib/erl_interface/test/all_SUITE_data/runner.c index 24df0f5f40..a474c17722 100644 --- a/lib/erl_interface/test/all_SUITE_data/runner.c +++ b/lib/erl_interface/test/all_SUITE_data/runner.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -18,6 +18,7 @@ */ #include <stdio.h> +#include <stdlib.h> #include <errno.h> #include <sys/types.h> #include <sys/stat.h> diff --git a/lib/erl_interface/test/ei_decode_SUITE.erl b/lib/erl_interface/test/ei_decode_SUITE.erl index 5a94994912..d7df63cc88 100644 --- a/lib/erl_interface/test/ei_decode_SUITE.erl +++ b/lib/erl_interface/test/ei_decode_SUITE.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2012. All Rights Reserved. +%% Copyright Ericsson AB 2004-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c index f5c8c4fa7d..a676e59470 100644 --- a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c +++ b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/erl_interface/test/ei_encode_SUITE.erl b/lib/erl_interface/test/ei_encode_SUITE.erl index e0ac4c0637..8d622a9e65 100644 --- a/lib/erl_interface/test/ei_encode_SUITE.erl +++ b/lib/erl_interface/test/ei_encode_SUITE.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2012. All Rights Reserved. +%% Copyright Ericsson AB 2004-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c b/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c index e904375e9e..50fcaace8d 100644 --- a/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c +++ b/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/erl_interface/test/ei_format_SUITE_data/ei_format_test.c b/lib/erl_interface/test/ei_format_SUITE_data/ei_format_test.c index 4f6c15ba9c..2f53effd78 100644 --- a/lib/erl_interface/test/ei_format_SUITE_data/ei_format_test.c +++ b/lib/erl_interface/test/ei_format_SUITE_data/ei_format_test.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -22,6 +22,7 @@ #endif #include "ei_runner.h" +#include <string.h> /* * Purpose: Tests the ei_format() function. diff --git a/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c b/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c index cc9b8048ca..8ce89e237a 100644 --- a/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c +++ b/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -19,6 +19,9 @@ #include "ei_runner.h" +#include <string.h> +#include <stdlib.h> + /* * Purpose: Tests the ei_print() function. * Author: Jakob diff --git a/lib/erl_interface/test/erl_eterm_SUITE_data/print_term.c b/lib/erl_interface/test/erl_eterm_SUITE_data/print_term.c index 56e2d43d2f..fd91373c3e 100644 --- a/lib/erl_interface/test/erl_eterm_SUITE_data/print_term.c +++ b/lib/erl_interface/test/erl_eterm_SUITE_data/print_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,6 +23,7 @@ */ #include <stdio.h> +#include <stdlib.h> #include <errno.h> #include <sys/types.h> #include <sys/stat.h> diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index 2a3843bda6..06ea973d9a 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1 +1 @@ -EI_VSN = 3.7.10 +EI_VSN = 3.7.11 diff --git a/lib/et/doc/src/notes.xml b/lib/et/doc/src/notes.xml index af2f761fc2..8d9675b801 100644 --- a/lib/et/doc/src/notes.xml +++ b/lib/et/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2002</year><year>2012</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/eunit/doc/src/notes.xml b/lib/eunit/doc/src/notes.xml index 8d10cc7548..fd51b05f79 100644 --- a/lib/eunit/doc/src/notes.xml +++ b/lib/eunit/doc/src/notes.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2008</year> - <year>2012</year> + <year>2013</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> diff --git a/lib/gs/doc/src/notes.xml b/lib/gs/doc/src/notes.xml index 2d68665b06..f778c010e9 100644 --- a/lib/gs/doc/src/notes.xml +++ b/lib/gs/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 028676bfd3..0a2c6e822f 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -1071,7 +1071,7 @@ type(hipe_bifs, ref_set, 2, Xs) -> strict(arg_types(hipe_bifs, ref_set, 2), Xs, fun (_) -> t_nil() end); type(hipe_bifs, remove_refs_from, 1, Xs) -> strict(arg_types(hipe_bifs, remove_refs_from, 1), Xs, - fun (_) -> t_nil() end); + fun (_) -> t_atom('ok') end); type(hipe_bifs, set_funinfo_native_address, 3, Xs) -> strict(arg_types(hipe_bifs, set_funinfo_native_address, 3), Xs, fun (_) -> t_nil() end); diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index 532b2e43cd..d1243b2325 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -205,7 +205,7 @@ t_var/1, t_var_name/1, %% t_assign_variables_to_subtype/2, - type_is_defined/3, + type_is_defined/4, record_field_diffs_to_string/2, subst_all_vars_to_any/1, lift_list_to_pos_empty/1, @@ -544,12 +544,12 @@ t_opaque_from_records(RecDict) -> OpaqueRecDict = dict:filter(fun(Key, _Value) -> case Key of - {opaque, _Name} -> true; + {opaque, _Name, _Arity} -> true; _ -> false end end, RecDict), OpaqueTypeDict = - dict:map(fun({opaque, Name}, {Module, Type, ArgNames}) -> + dict:map(fun({opaque, Name, _Arity}, {Module, Type, ArgNames}) -> case ArgNames of [] -> t_opaque(Module, Name, [], t_from_form(Type, RecDict)); @@ -707,8 +707,8 @@ t_solve_remote_type(#remote{mod = RemMod, name = Name, args = Args} = RemType, MFA = {RemMod, Name, ArgsLen}, case sets:is_element(MFA, ET) of true -> - case lookup_type(Name, RemDict) of - {type, {_Mod, Type, ArgNames}} when ArgsLen =:= length(ArgNames) -> + case lookup_type(Name, ArgsLen, RemDict) of + {type, {_Mod, Type, ArgNames}} -> {NewType, NewCycle, NewRR} = case can_unfold_more(RemType, C) of true -> @@ -726,7 +726,7 @@ t_solve_remote_type(#remote{mod = RemMod, name = Name, args = Args} = RemType, false -> RT end, {RT1, RetRR}; - {opaque, {Mod, Type, ArgNames}} when ArgsLen =:= length(ArgNames) -> + {opaque, {Mod, Type, ArgNames}} -> List = lists:zip(ArgNames, Args), TmpVarDict = dict:from_list(List), {Rep, NewCycle, NewRR} = @@ -746,12 +746,6 @@ t_solve_remote_type(#remote{mod = RemMod, name = Name, args = Args} = RemType, {t_from_form({opaque, -1, Name, {Mod, Args, RT1}}, RemDict, TmpVarDict), RetRR}; - {type, _} -> - Msg = io_lib:format("Unknown remote type ~w\n", [Name]), - throw({error, Msg}); - {opaque, _} -> - Msg = io_lib:format("Unknown remote opaque type ~w\n", [Name]), - throw({error, Msg}); error -> Msg = io_lib:format("Unable to find remote type ~w:~w()\n", [RemMod, Name]), @@ -3241,6 +3235,8 @@ t_to_string(?bitstr(0, 0), _RecDict) -> "<<>>"; t_to_string(?bitstr(8, 0), _RecDict) -> "binary()"; +t_to_string(?bitstr(1, 0), _RecDict) -> + "bitstring()"; t_to_string(?bitstr(0, B), _RecDict) -> lists:flatten(io_lib:format("<<_:~w>>", [B])); t_to_string(?bitstr(U, 0), _RecDict) -> @@ -3680,8 +3676,9 @@ t_from_form({type, _L, union, Args}, TypeNames, InOpaque, RecDict, VarDict) -> {L, R} = list_from_form(Args, TypeNames, InOpaque, RecDict, VarDict), {t_sup(L), R}; t_from_form({type, _L, Name, Args}, TypeNames, InOpaque, RecDict, VarDict) -> - case lookup_type(Name, RecDict) of - {type, {_Module, Type, ArgNames}} when length(Args) =:= length(ArgNames) -> + ArgsLen = length(Args), + case lookup_type(Name, ArgsLen, RecDict) of + {type, {_Module, Type, ArgNames}} -> case can_unfold_more({type, Name}, TypeNames) of true -> List = lists:zipwith( @@ -3701,7 +3698,7 @@ t_from_form({type, _L, Name, Args}, TypeNames, InOpaque, RecDict, VarDict) -> end; false -> {t_any(), [{type, Name}]} end; - {opaque, {Module, Type, ArgNames}} when length(Args) =:= length(ArgNames) -> + {opaque, {Module, Type, ArgNames}} -> {Rep, Rret} = case can_unfold_more({opaque, Name}, TypeNames) of true -> @@ -3730,12 +3727,9 @@ t_from_form({type, _L, Name, Args}, TypeNames, InOpaque, RecDict, VarDict) -> RecDict, VarDict) end, {Tret, Rret}; - {type, _} -> - throw({error, io_lib:format("Unknown type ~w\n", [Name])}); - {opaque, _} -> - throw({error, io_lib:format("Unknown opaque type ~w\n", [Name])}); error -> - throw({error, io_lib:format("Unable to find type ~w\n", [Name])}) + Msg = io_lib:format("Unable to find type ~w/~w\n", [Name, ArgsLen]), + throw({error, Msg}) end; t_from_form({opaque, _L, Name, {Mod, Args, Rep}}, _TypeNames, _InOpaque, _RecDict, _VarDict) -> @@ -3870,12 +3864,14 @@ t_form_to_string({type, _L, binary, [Base, Unit]} = Type) -> case {U, B} of {0, 0} -> "<<>>"; {8, 0} -> "binary()"; + {1, 0} -> "bitstring()"; {0, B} -> lists:flatten(io_lib:format("<<_:~w>>", [B])); {U, 0} -> lists:flatten(io_lib:format("<<_:_*~w>>", [U])); {U, B} -> lists:flatten(io_lib:format("<<_:~w,_:_*~w>>", [B, U])) end; _ -> io_lib:format("Badly formed bitstr type ~w", [Type]) end; +t_form_to_string({type, _L, bitstring, []}) -> "bitstring()"; t_form_to_string({type, _L, 'fun', []}) -> "fun()"; t_form_to_string({type, _L, 'fun', [{type, _, any}, Range]}) -> "fun(...) -> " ++ t_form_to_string(Range); @@ -3986,20 +3982,20 @@ lookup_record(Tag, Arity, RecDict) when is_atom(Tag) -> error -> error end. -lookup_type(Name, RecDict) -> - case dict:find({type, Name}, RecDict) of +lookup_type(Name, Arity, RecDict) -> + case dict:find({type, Name, Arity}, RecDict) of error -> - case dict:find({opaque, Name}, RecDict) of + case dict:find({opaque, Name, Arity}, RecDict) of error -> error; {ok, Found} -> {opaque, Found} end; {ok, Found} -> {type, Found} end. --spec type_is_defined('type' | 'opaque', atom(), dict()) -> boolean(). +-spec type_is_defined('type' | 'opaque', atom(), arity(), dict()) -> boolean(). -type_is_defined(TypeOrOpaque, Name, RecDict) -> - dict:is_key({TypeOrOpaque, Name}, RecDict). +type_is_defined(TypeOrOpaque, Name, Arity, RecDict) -> + dict:is_key({TypeOrOpaque, Name, Arity}, RecDict). can_unfold_more(TypeName, TypeNames) -> Fun = fun(E, Acc) -> case E of TypeName -> Acc + 1; _ -> Acc end end, diff --git a/lib/hipe/doc/src/hipe_app.xml b/lib/hipe/doc/src/hipe_app.xml index 56729d4cc4..9a1aa943d4 100644 --- a/lib/hipe/doc/src/hipe_app.xml +++ b/lib/hipe/doc/src/hipe_app.xml @@ -21,7 +21,7 @@ </legalnotice> - <title>snmp</title> + <title>HiPE</title> <prepared></prepared> <responsible></responsible> <docno></docno> diff --git a/lib/hipe/doc/src/notes.xml b/lib/hipe/doc/src/notes.xml index 4be1d451ff..73ab1dbfda 100644 --- a/lib/hipe/doc/src/notes.xml +++ b/lib/hipe/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2012</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,32 @@ </header> <p>This document describes the changes made to HiPE.</p> +<section><title>Hipe 3.10.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Bug fixed in hipe to where it did not allow unicode code + points 16#FFFE and 16#FFFF in bit syntax in natively + compiled modules.</p> + <p> + Own Id: OTP-10867</p> + </item> + <item> + <p> + Fix bug in hipe compiled code related to the handling of + <c>is_number/1</c>. (Thanks to Sebastian Egner and + Johannes Wei�l for minimal test code and Kostis for quick + patch)</p> + <p> + Own Id: OTP-10897</p> + </item> + </list> + </section> + +</section> + <section><title>Hipe 3.10</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/hipe/icode/hipe_icode_range.erl b/lib/hipe/icode/hipe_icode_range.erl index c222e8a5d5..46c5a39f2c 100644 --- a/lib/hipe/icode/hipe_icode_range.erl +++ b/lib/hipe/icode/hipe_icode_range.erl @@ -784,6 +784,8 @@ analyse_type(Type, Info, Rewrite) -> integer -> TrueRange = inf(any_range(), OldVarRange), FalseRange = inf(none_range(), OldVarRange); + number -> + TrueRange = FalseRange = OldVarRange; _ -> TrueRange = inf(none_range(), OldVarRange), FalseRange = OldVarRange diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index b2789978a4..6e00b13292 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -1135,7 +1135,7 @@ option_text(debug) -> option_text(icode_range) -> "Performs integer range analysis on the Icode level"; option_text(icode_ssa_check) -> - "Checks whether Icode is on SSA form or not\n"; + "Checks whether Icode is on SSA form or not"; option_text(icode_ssa_copy_prop) -> "Performs copy propagation on Icode SSA"; option_text(icode_ssa_const_prop) -> @@ -1143,14 +1143,14 @@ option_text(icode_ssa_const_prop) -> option_text(icode_ssa_struct_reuse) -> "Factors out common tuple and list constructions on Icode SSA"; option_text(icode_type) -> - "Performs type analysis on the Icode level" ++ + "Performs type analysis on the Icode level\n" ++ "and then simplifies the code based on the results of this analysis"; option_text(load) -> "Automatically load the produced native code into memory"; option_text(peephole) -> "Enables peephole optimizations"; option_text(pmatch) -> - "Enables pattern matching compilation when compiling from Core; " ++ + "Enables pattern matching compilation when compiling from Core;\n" ++ "has no effect when compiling from BEAM bytecode"; option_text(pp_asm) -> "Displays assembly listing with addresses and bytecode\n" ++ @@ -1197,11 +1197,11 @@ option_text(timeout) -> " The limit must be a non-negative integer or the atom 'infinity'.\n" ++ " The current default limit is 15 minutes (900000 ms)."; option_text(use_indexing) -> - "Use indexing for multiple-choice branch selection."; + "Use indexing for multiple-choice branch selection"; option_text(use_callgraph) -> - "Compile the functions in a module according to a reversed topological " ++ - "sorted order to gain more information when using a persistent lookup " ++ - "table for storing intra-modular type information."; + "Compile the functions in a module according to a reversed topological\n" ++ + "sorted order to gain more information when using a persistent lookup\n" ++ + "table for storing intra-modular type information"; option_text(verbose) -> "Output information about what is being done"; option_text(Opt) when is_atom(Opt) -> diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index d60492ac46..6cd87708ef 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.10 +HIPE_VSN = 3.10.1 diff --git a/lib/ic/c_src/ic.c b/lib/ic/c_src/ic.c index 796842f4f8..1c9b58b21b 100644 --- a/lib/ic/c_src/ic.c +++ b/lib/ic/c_src/ic.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -149,7 +149,6 @@ void ic_init_ref(CORBA_Environment *env, erlang_ref *ref) { strcpy(ref->node, erl_thisnodename()); - ref->node_org_enc = ERLANG_LATIN1; ref->len = 3; diff --git a/lib/ic/doc/src/notes.xml b/lib/ic/doc/src/notes.xml index 8c5634d440..92cc7cab7a 100644 --- a/lib/ic/doc/src/notes.xml +++ b/lib/ic/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1998</year><year>2012</year> + <year>1998</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,7 +30,25 @@ <file>notes.xml</file> </header> - <section><title>IC 4.3</title> + <section><title>IC 4.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Revert the structs <c>erlang_pid</c>, <c>erlang_port</c> + and <c>erlang_ref</c> as they were before R16A (without + <c>node_org_enc</c>) in order to be backward compatible + with user code that accesses the fields of these structs.</p> + <p> + Own Id: OTP-10885 Aux Id: seq12256 </p> + </item> + </list> + </section> + +</section> + +<section><title>IC 4.3</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/ic/examples/all-against-all/client.c b/lib/ic/examples/all-against-all/client.c index 022b9fd1c0..a638ac6b86 100644 --- a/lib/ic/examples/all-against-all/client.c +++ b/lib/ic/examples/all-against-all/client.c @@ -88,7 +88,6 @@ int main(){ /* Initiating pid*/ strcpy(pid.node,client_node); - pid.node_org_enc = ERLANG_LATIN1; pid.num = 99; pid.serial = 0; pid.creation = 0; diff --git a/lib/ic/examples/c-client/client.c b/lib/ic/examples/c-client/client.c index 3e9678ae4d..53dbbf9192 100644 --- a/lib/ic/examples/c-client/client.c +++ b/lib/ic/examples/c-client/client.c @@ -64,7 +64,6 @@ int main() /* Initiating pid*/ strcpy(pid.node,CLNODE); - pid.node_org_enc = ERLANG_LATIN1; pid.num = 99; pid.serial = 0; pid.creation = 0; diff --git a/lib/ic/examples/c-server/client.c b/lib/ic/examples/c-server/client.c index b6c7ef6bce..d683d586d8 100644 --- a/lib/ic/examples/c-server/client.c +++ b/lib/ic/examples/c-server/client.c @@ -58,7 +58,6 @@ int main() /* Initiating pid*/ strcpy(pid.node, CLNODE); - pid.node_org_enc = ERLANG_LATIN1; pid.num = 99; pid.serial = 0; pid.creation = 0; diff --git a/lib/ic/java_src/com/ericsson/otp/ic/Any.java b/lib/ic/java_src/com/ericsson/otp/ic/Any.java index c7ab6bad84..003ad5dd93 100644 --- a/lib/ic/java_src/com/ericsson/otp/ic/Any.java +++ b/lib/ic/java_src/com/ericsson/otp/ic/Any.java @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ic/java_src/com/ericsson/otp/ic/Environment.java b/lib/ic/java_src/com/ericsson/otp/ic/Environment.java index 4c5315ee98..fff854a5f8 100644 --- a/lib/ic/java_src/com/ericsson/otp/ic/Environment.java +++ b/lib/ic/java_src/com/ericsson/otp/ic/Environment.java @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ic/java_src/com/ericsson/otp/ic/Term.java b/lib/ic/java_src/com/ericsson/otp/ic/Term.java index 611f1f04d7..510e3b6644 100644 --- a/lib/ic/java_src/com/ericsson/otp/ic/Term.java +++ b/lib/ic/java_src/com/ericsson/otp/ic/Term.java @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ic/java_src/com/ericsson/otp/ic/TermHelper.java b/lib/ic/java_src/com/ericsson/otp/ic/TermHelper.java index 5675eee706..2942480283 100644 --- a/lib/ic/java_src/com/ericsson/otp/ic/TermHelper.java +++ b/lib/ic/java_src/com/ericsson/otp/ic/TermHelper.java @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ic/java_src/com/ericsson/otp/ic/TypeCode.java b/lib/ic/java_src/com/ericsson/otp/ic/TypeCode.java index 327864320f..734872386b 100644 --- a/lib/ic/java_src/com/ericsson/otp/ic/TypeCode.java +++ b/lib/ic/java_src/com/ericsson/otp/ic/TypeCode.java @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c b/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c index 98fcdcc60f..f910592609 100644 --- a/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c +++ b/lib/ic/test/c_client_erl_server_SUITE_data/c_client.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -390,7 +390,6 @@ int main(int argc, char **argv) env->_from_pid = &pid; strcpy(pid.node, this_node); - pid.node_org_enc = ERLANG_LATIN1; pid.num = fd; pid.serial = 0; pid.creation = 0; @@ -1007,7 +1006,7 @@ static int string4_test(IC_Env *env) static int pid_test(IC_Env *env) { - erlang_pid pid = {"", ERLANG_LATIN1, 7, 0, 0}, pido, pidr; + erlang_pid pid = {"", 7, 0, 0}, pido, pidr; strcpy(pid.node, this_node), /* this currently running node */ fprintf(stdout, "\n======== m_i_pid test ======\n\n"); @@ -1031,7 +1030,7 @@ static int pid_test(IC_Env *env) static int port_test(IC_Env *env) { - erlang_port porti = {"node", ERLANG_LATIN1, 5, 1}, porto, portr; + erlang_port porti = {"node", 5, 1}, porto, portr; fprintf(stdout, "\n======== m_i_port test ======\n\n"); portr = m_i_port_test(NULL, &porti, &porto, env); @@ -1054,7 +1053,7 @@ static int port_test(IC_Env *env) static int ref_test(IC_Env *env) { - erlang_ref refi = { "node1", ERLANG_UTF8, 3, {1, 2, 3}, 1}, + erlang_ref refi = { "node1", 3, {1, 2, 3}, 1}, refo, refr; fprintf(stdout, "\n======== m_i_ref test ======\n\n"); @@ -1112,7 +1111,6 @@ static int typedef_test(IC_Env *env) long tl; strcpy(mbi.node,"node"); - mbi.node_org_enc = ERLANG_LATIN1; mbi.id = 15; mbi.creation = 1; diff --git a/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c b/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c index 4ced4fb5e5..40ff898dcb 100644 --- a/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c +++ b/lib/ic/test/c_client_erl_server_proto_SUITE_data/c_client.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -393,7 +393,6 @@ int main(int argc, char **argv) env->_from_pid = &pid; strcpy(pid.node, this_node); - pid.node_org_enc = ERLANG_LATIN1; pid.num = fd; pid.serial = 0; pid.creation = 0; @@ -1010,7 +1009,7 @@ static int string4_test(IC_Env *env) static int pid_test(IC_Env *env) { - erlang_pid pid = {"", ERLANG_LATIN1, 7, 0, 0}, pido, pidr; + erlang_pid pid = {"", 7, 0, 0}, pido, pidr; strcpy(pid.node, this_node), /* this currently running node */ fprintf(stdout, "\n======== m_i_pid test ======\n\n"); @@ -1034,7 +1033,7 @@ static int pid_test(IC_Env *env) static int port_test(IC_Env *env) { - erlang_port porti = {"node", ERLANG_LATIN1, 5, 1}, porto, portr; + erlang_port porti = {"node", 5, 1}, porto, portr; fprintf(stdout, "\n======== m_i_port test ======\n\n"); portr = m_i_port_test(NULL, &porti, &porto, env); @@ -1057,7 +1056,7 @@ static int port_test(IC_Env *env) static int ref_test(IC_Env *env) { - erlang_ref refi = { "node1", ERLANG_LATIN1, 3, {1, 2, 3}, 1}, + erlang_ref refi = { "node1", 3, {1, 2, 3}, 1}, refo, refr; fprintf(stdout, "\n======== m_i_ref test ======\n\n"); @@ -1115,7 +1114,6 @@ static int typedef_test(IC_Env *env) long tl; strcpy(mbi.node,"node"); - mbi.node_org_enc = ERLANG_LATIN1; mbi.id = 15; mbi.creation = 1; diff --git a/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c b/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c index 7d5abcc376..f270d2abb9 100644 --- a/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c +++ b/lib/ic/test/c_client_erl_server_proto_tmo_SUITE_data/c_client.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -393,7 +393,6 @@ int main(int argc, char **argv) env->_from_pid = &pid; strcpy(pid.node, this_node); - pid.node_org_enc = ERLANG_LATIN1; pid.num = fd; pid.serial = 0; pid.creation = 0; @@ -1010,7 +1009,7 @@ static int string4_test(IC_Env *env) static int pid_test(IC_Env *env) { - erlang_pid pid = {"", ERLANG_LATIN1, 7, 0, 0}, pido, pidr; + erlang_pid pid = {"", 7, 0, 0}, pido, pidr; strcpy(pid.node, this_node), /* this currently running node */ fprintf(stdout, "\n======== m_i_pid test ======\n\n"); @@ -1034,7 +1033,7 @@ static int pid_test(IC_Env *env) static int port_test(IC_Env *env) { - erlang_port porti = {"node", ERLANG_LATIN1, 5, 1}, porto, portr; + erlang_port porti = {"node", 5, 1}, porto, portr; fprintf(stdout, "\n======== m_i_port test ======\n\n"); portr = m_i_port_test(NULL, &porti, &porto, env); @@ -1057,7 +1056,7 @@ static int port_test(IC_Env *env) static int ref_test(IC_Env *env) { - erlang_ref refi = { "node1", ERLANG_LATIN1, 3, {1, 2, 3}, 1}, + erlang_ref refi = { "node1", 3, {1, 2, 3}, 1}, refo, refr; fprintf(stdout, "\n======== m_i_ref test ======\n\n"); @@ -1115,7 +1114,6 @@ static int typedef_test(IC_Env *env) long tl; strcpy(mbi.node,"node"); - mbi.node_org_enc = ERLANG_LATIN1; mbi.id = 15; mbi.creation = 1; diff --git a/lib/ic/test/erl_client_c_server_SUITE_data/callbacks.c b/lib/ic/test/erl_client_c_server_SUITE_data/callbacks.c index ed21ba7baf..3cc71aa2e8 100644 --- a/lib/ic/test/erl_client_c_server_SUITE_data/callbacks.c +++ b/lib/ic/test/erl_client_c_server_SUITE_data/callbacks.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -494,12 +494,10 @@ m_i_port_test__rs* m_i_port_test__cb(CORBA_Object oe_obj, m_i_port_test__rs* rs = NULL; strcpy((*a).node,(*b).node); - a->node_org_enc = b->node_org_enc; (*a).id = (*b).id; (*a).creation = 0; strcpy((*c).node,(*b).node); - c->node_org_enc = b->node_org_enc; (*c).id = (*b).id; (*c).creation = 0; return rs; @@ -516,7 +514,6 @@ m_i_ref_test__rs* m_i_ref_test__cb(CORBA_Object oe_obj, m_i_ref_test__rs* rs = NULL; strcpy((*a).node,(*b).node); - a->node_org_enc = b->node_org_enc; /*(*a).id = (*b).id;*/ (*a).len = (*b).len; (*a).n[0] = (*b).n[0]; @@ -525,7 +522,6 @@ m_i_ref_test__rs* m_i_ref_test__cb(CORBA_Object oe_obj, (*a).creation = 0; strcpy((*c).node,(*b).node); - c->node_org_enc = b->node_org_enc; /*(*c).id = (*b).id;*/ (*c).len = (*b).len; (*c).n[0] = (*b).n[0]; @@ -561,7 +557,6 @@ m_i_typedef_test__rs* m_i_typedef_test__cb(CORBA_Object oe_obj, *d = *b; strcpy((*e).node,(*c).node); - e->node_org_enc = c->node_org_enc; (*e).id = (*c).id; (*e).creation = 0; *a = 4711; diff --git a/lib/ic/test/erl_client_c_server_proto_SUITE_data/callbacks.c b/lib/ic/test/erl_client_c_server_proto_SUITE_data/callbacks.c index d813cae45a..96fac33b29 100644 --- a/lib/ic/test/erl_client_c_server_proto_SUITE_data/callbacks.c +++ b/lib/ic/test/erl_client_c_server_proto_SUITE_data/callbacks.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2011. All Rights Reserved. + * Copyright Ericsson AB 2004-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -494,12 +494,10 @@ m_i_port_test__rs* m_i_port_test__cb(CORBA_Object oe_obj, m_i_port_test__rs* rs = NULL; strcpy((*a).node,(*b).node); - a->node_org_enc = b->node_org_enc; (*a).id = (*b).id; (*a).creation = 0; strcpy((*c).node,(*b).node); - c->node_org_enc = b->node_org_enc; (*c).id = (*b).id; (*c).creation = 0; return rs; @@ -516,7 +514,6 @@ m_i_ref_test__rs* m_i_ref_test__cb(CORBA_Object oe_obj, m_i_ref_test__rs* rs = NULL; strcpy((*a).node,(*b).node); - a->node_org_enc = b->node_org_enc; /*(*a).id = (*b).id;*/ (*a).len = (*b).len; (*a).n[0] = (*b).n[0]; @@ -525,7 +522,6 @@ m_i_ref_test__rs* m_i_ref_test__cb(CORBA_Object oe_obj, (*a).creation = 0; strcpy((*c).node,(*b).node); - c->node_org_enc = b->node_org_enc; /*(*c).id = (*b).id;*/ (*c).len = (*b).len; (*c).n[0] = (*b).n[0]; @@ -561,7 +557,6 @@ m_i_typedef_test__rs* m_i_typedef_test__cb(CORBA_Object oe_obj, *d = *b; strcpy((*e).node,(*c).node); - e->node_org_enc = c->node_org_enc; (*e).id = (*c).id; (*e).creation = 0; *a = 4711; diff --git a/lib/ic/test/java_client_erl_server_SUITE_data/Makefile.src b/lib/ic/test/java_client_erl_server_SUITE_data/Makefile.src index 3143ab427b..89fca8d270 100644 --- a/lib/ic/test/java_client_erl_server_SUITE_data/Makefile.src +++ b/lib/ic/test/java_client_erl_server_SUITE_data/Makefile.src @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2012. All Rights Reserved. +# Copyright Ericsson AB 2003-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ic/vsn.mk b/lib/ic/vsn.mk index dcaf81d26f..9fd8aedb63 100644 --- a/lib/ic/vsn.mk +++ b/lib/ic/vsn.mk @@ -1 +1 @@ -IC_VSN = 4.3 +IC_VSN = 4.3.1 diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index 3fced5dfcd..8438961511 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1997</year><year>2012</year> + <year>1997</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -163,11 +163,9 @@ </item> <marker id="prop_socket_type"></marker> - <tag>{socket_type, ip_comm | ssl | essl}</tag> + <tag>{socket_type, ip_comm | {essl, Config::proplist()}}</tag> <item> - <p>When using ssl, there are currently only one alternative. - <c>essl</c> specifically uses the Erlang based SSL. - <c>ssl</c> defaults to <c>essl</c>. </p> + <p> For ssl configuration options see <seealso marker="ssl:ssl#listen-2">ssl:listen/2</seealso> </p> <p>Defaults to <c>ip_comm</c>. </p> </item> @@ -395,71 +393,7 @@ bytes </item> </taglist> - - <marker id="props_ssl"></marker> - <p><em>ssl properties</em></p> - <taglist> - <marker id="prop_ssl_ca_cert_file"></marker> - <tag>{ssl_ca_certificate_file, path()}</tag> - <item> - <p>Used as cacertfile option in ssl:listen/2 see - <seealso marker="ssl:ssl">ssl(3)</seealso>. </p> - </item> - - <marker id="prop_ssl_cert_file"></marker> - <tag>{ssl_certificate_file, path()}</tag> - <item> - <p>Used as certfile option in ssl:listen/2 see - <seealso marker="ssl:ssl">ssl(3)</seealso>. </p> - </item> - - <marker id="prop_ssl_ciphers"></marker> - <tag>{ssl_ciphers, list()}</tag> - <item> - <p>Used as ciphers option in ssl:listen/2 see - <seealso marker="ssl:ssl">ssl(3)</seealso>. </p> - </item> - - <marker id="prop_ssl_verify_client"></marker> - <tag>{ssl_verify_client, integer()}</tag> - <item> - <p>Used as verify option in ssl:listen/2 see - <seealso marker="ssl:ssl">ssl(3)</seealso>. </p> - </item> - - <marker id="prop_ssl_verify_depth"></marker> - <tag>{ssl_verify_depth, integer()}</tag> - <item> - <p>Used as depth option in ssl:listen/2 see - <seealso marker="ssl:ssl">ssl(3)</seealso>. </p> - </item> - - <marker id="prop_ssl_passwd_callback_funct"></marker> - <tag>{ssl_password_callback_function, atom()}</tag> - <item> - <p>Used together with ssl_password_callback_module - to retrieve a value to use as password option to ssl:listen/2 - see <seealso marker="ssl:ssl">ssl(3)</seealso>. </p> - </item> - - <marker id="prop_ssl_passwd_callback_args"></marker> - <tag>{ssl_password_callback_arguments, list()}</tag> - <item> - <p>Used together with ssl_password_callback_function to supply a - list of arguments to the callback function. If not specified - the callback function will be assumed to have arity 0. </p> - </item> - - <marker id="prop_ssl_passwd_callback_mod"></marker> - <tag>{ssl_password_callback_module, atom()}</tag> - <item> - <p>Used together with ssl_password_callback_function - to retrieve a value to use as password option to ssl:listen/2 - see <seealso marker="ssl:ssl">ssl(3)</seealso>. </p> - </item> - - </taglist> - + <marker id="props_alias"></marker> <p><em>URL aliasing properties - requires mod_alias</em></p> <taglist> diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 0236a0bf51..525beecd31 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2002</year><year>2012</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -33,7 +33,47 @@ </header> - <section><title>Inets 5.9.3</title> + <section><title>Inets 5.9.4</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + httpd: The modules option now defaults to the documented + value.</p> + <p> + Own Id: OTP-10844</p> + </item> + <item> + <p> + httpc: Fixed persistent connection implementation that + was broken by a patch to R13. The patch made persisten + connections behaved the same way as pipelining.</p> + <p> + Own Id: OTP-10845</p> + </item> + <item> + <p> + httpd: Simplified configuration of ssl in httpd, this + also enables all ssl options to be configured. The old + and limited way is no longer documented but will be + supported for backwards comatibility for some time.</p> + <p> + Own Id: OTP-10846</p> + </item> + <item> + <p> + Handle correctly the "No files found or file unavailable" + error code. Thanks to Serge Aleynikov</p> + <p> + Own Id: OTP-10886</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 5.9.3</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index fe25c23316..132a384a49 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -1899,6 +1899,10 @@ ctrl_result_response(pos_compl, #state{client = From} = State, _) -> gen_server:reply(From, ok), {noreply, State#state{client = undefined, caller = undefined}}; +ctrl_result_response(enofile, #state{client = From} = State, _) -> + gen_server:reply(From, {error, enofile}), + {noreply, State#state{client = undefined, caller = undefined}}; + ctrl_result_response(Status, #state{client = From} = State, _) when (Status =:= etnospc) orelse (Status =:= epnospc) orelse diff --git a/lib/inets/src/ftp/ftp_response.erl b/lib/inets/src/ftp/ftp_response.erl index faeacb31ab..4bf788e946 100644 --- a/lib/inets/src/ftp/ftp_response.erl +++ b/lib/inets/src/ftp/ftp_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. +%% Copyright Ericsson AB 2005-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -162,6 +162,7 @@ error_string(epath) -> "No such file or directory, already exists, " error_string(etype) -> "No such type."; error_string(euser) -> "User name or password not valid."; error_string(etnospc) -> "Insufficient storage space in system."; +error_string(enofile) -> "No files found or file unavailable"; error_string(epnospc) -> "Exceeded storage allocation " "(for current directory or dataset)."; error_string(efnamena) -> "File name not allowed."; @@ -180,6 +181,8 @@ interpret_status(?POS_COMPL,_,_) -> pos_compl; interpret_status(?POS_INTERM,?AUTH_ACC,2) -> pos_interm_acct; %% Positive Intermediate Reply interpret_status(?POS_INTERM,_,_) -> pos_interm; +%% No files found or file not available +interpret_status(?TRANS_NEG_COMPL,?FILE_SYSTEM,0) -> enofile; %% No storage area no action taken interpret_status(?TRANS_NEG_COMPL,?FILE_SYSTEM,2) -> etnospc; %% Temporary Error, no action taken diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 784a9c0019..857043bae2 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2012. All Rights Reserved. +%% Copyright Ericsson AB 2002-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -257,7 +257,7 @@ handle_call(#request{address = Addr} = Request, _, session = #session{type = pipeline} = Session, timers = Timers, options = #options{proxy = Proxy} = _Options, - profile_name = ProfileName} = State) + profile_name = ProfileName} = State0) when Status =/= undefined -> ?hcrv("new request on a pipeline session", @@ -274,18 +274,18 @@ handle_call(#request{address = Addr} = Request, _, ?hcrd("request sent", []), %% Activate the request time out for the new request - NewState = - activate_request_timeout(State#state{request = Request}), + State1 = + activate_request_timeout(State0#state{request = Request}), ClientClose = httpc_request:is_client_closing(Request#request.headers), - case State#state.request of - #request{} -> %% Old request not yet finished + case State0#state.request of + #request{} = OldRequest -> %% Old request not yet finished ?hcrd("old request still not finished", []), %% Make sure to use the new value of timers in state - NewTimers = NewState#state.timers, - NewPipeline = queue:in(Request, State#state.pipeline), + NewTimers = State1#state.timers, + NewPipeline = queue:in(Request, State1#state.pipeline), NewSession = Session#session{queue_length = %% Queue + current @@ -293,9 +293,11 @@ handle_call(#request{address = Addr} = Request, _, client_close = ClientClose}, insert_session(NewSession, ProfileName), ?hcrd("session updated", []), - {reply, ok, State#state{pipeline = NewPipeline, - session = NewSession, - timers = NewTimers}}; + {reply, ok, State1#state{ + request = OldRequest, + pipeline = NewPipeline, + session = NewSession, + timers = NewTimers}}; undefined -> %% Note: tcp-message receiving has already been %% activated by handle_pipeline/2. @@ -306,20 +308,15 @@ handle_call(#request{address = Addr} = Request, _, Session#session{queue_length = 1, client_close = ClientClose}, httpc_manager:insert_session(NewSession, ProfileName), - Relaxed = - (Request#request.settings)#http_options.relaxed, - MFA = {httpc_response, parse, - [State#state.max_header_size, Relaxed]}, NewTimers = Timers#timers{queue_timer = undefined}, ?hcrd("session created", []), - {reply, ok, NewState#state{request = Request, - session = NewSession, - mfa = MFA, - timers = NewTimers}} + State = init_wait_for_response_state(Request, State1#state{session = NewSession, + timers = NewTimers}), + {reply, ok, State} end; {error, Reason} -> ?hcri("failed sending request", [{reason, Reason}]), - {reply, {pipeline_failed, Reason}, State} + {reply, {pipeline_failed, Reason}, State0} end; handle_call(#request{address = Addr} = Request, _, @@ -327,7 +324,7 @@ handle_call(#request{address = Addr} = Request, _, session = #session{type = keep_alive} = Session, timers = Timers, options = #options{proxy = Proxy} = _Options, - profile_name = ProfileName} = State) + profile_name = ProfileName} = State0) when Status =/= undefined -> ?hcrv("new request on a keep-alive session", @@ -335,65 +332,54 @@ handle_call(#request{address = Addr} = Request, _, {profile, ProfileName}, {status, Status}]), - Address = handle_proxy(Addr, Proxy), - case httpc_request:send(Address, Session, Request) of - ok -> - - ?hcrd("request sent", []), - - %% Activate the request time out for the new request - NewState = - activate_request_timeout(State#state{request = Request}), - - ClientClose = - httpc_request:is_client_closing(Request#request.headers), - - case State#state.request of - #request{} -> %% Old request not yet finished - %% Make sure to use the new value of timers in state - ?hcrd("old request still not finished", []), - NewTimers = NewState#state.timers, - NewKeepAlive = queue:in(Request, State#state.keep_alive), - NewSession = - Session#session{queue_length = - %% Queue + current - queue:len(NewKeepAlive) + 1, - client_close = ClientClose}, - insert_session(NewSession, ProfileName), - ?hcrd("session updated", []), - {reply, ok, State#state{keep_alive = NewKeepAlive, - session = NewSession, - timers = NewTimers}}; - undefined -> - %% Note: tcp-message reciving has already been - %% activated by handle_pipeline/2. - ?hcrd("no current request", []), - cancel_timer(Timers#timers.queue_timer, - timeout_queue), - NewSession = - Session#session{queue_length = 1, - client_close = ClientClose}, - insert_session(NewSession, ProfileName), - Relaxed = - (Request#request.settings)#http_options.relaxed, - MFA = {httpc_response, parse, - [State#state.max_header_size, Relaxed]}, - {reply, ok, NewState#state{request = Request, - session = NewSession, - mfa = MFA}} - end; - - {error, Reason} -> - ?hcri("failed sending request", [{reason, Reason}]), - {reply, {request_failed, Reason}, State} + ClientClose = httpc_request:is_client_closing(Request#request.headers), + + case State0#state.request of + #request{} -> %% Old request not yet finished + %% Make sure to use the new value of timers in state + ?hcrd("old request still not finished", []), + NewKeepAlive = queue:in(Request, State0#state.keep_alive), + NewSession = + Session#session{queue_length = + %% Queue + current + queue:len(NewKeepAlive) + 1, + client_close = ClientClose}, + insert_session(NewSession, ProfileName), + ?hcrd("session updated", []), + {reply, ok, State0#state{keep_alive = NewKeepAlive, + session = NewSession}}; + undefined -> + %% Note: tcp-message reciving has already been + %% activated by handle_pipeline/2. + ?hcrd("no current request", []), + cancel_timer(Timers#timers.queue_timer, + timeout_queue), + Address = handle_proxy(Addr, Proxy), + case httpc_request:send(Address, Session, Request) of + ok -> + ?hcrd("request sent", []), + + %% Activate the request time out for the new request + State1 = + activate_request_timeout(State0#state{request = Request}), + NewTimers = State1#state.timers, + NewSession = + Session#session{queue_length = 1, + client_close = ClientClose}, + insert_session(NewSession, ProfileName), + State = init_wait_for_response_state(Request, State1#state{session = NewSession, + timers = NewTimers}), + {reply, ok, State}; + {error, Reason} -> + ?hcri("failed sending request", [{reason, Reason}]), + {reply, {request_failed, Reason}, State0} + end end; - handle_call(info, _, State) -> Info = handler_info(State), {reply, Info, State}. - %%-------------------------------------------------------------------- %% Function: handle_cast(Msg, State) -> {noreply, State} | %% {noreply, State, Timeout} | @@ -1239,8 +1225,7 @@ handle_queue(#state{status = pipeline} = State, Data) -> handle_pipeline(#state{status = pipeline, session = Session, profile_name = ProfileName, - options = #options{pipeline_timeout = TimeOut}} = - State, + options = #options{pipeline_timeout = TimeOut}} = State, Data) -> ?hcrd("handle pipeline", [{profile, ProfileName}, @@ -1250,25 +1235,7 @@ handle_pipeline(#state{status = pipeline, case queue:out(State#state.pipeline) of {empty, _} -> ?hcrd("pipeline queue empty", []), - - %% The server may choose too teminate an idle pipeline - %% in this case we want to receive the close message - %% at once and not when trying to pipeline the next - %% request. - activate_once(Session), - - %% If a pipeline that has been idle for some time is not - %% closed by the server, the client may want to close it. - NewState = activate_queue_timeout(TimeOut, State), - update_session(ProfileName, Session, #session.queue_length, 0), - %% Note mfa will be initilized when a new request - %% arrives. - {noreply, - NewState#state{request = undefined, - mfa = undefined, - status_line = undefined, - headers = undefined, - body = undefined}}; + handle_empty_queue(Session, ProfileName, TimeOut, State); {{value, NextRequest}, Pipeline} -> ?hcrd("pipeline queue non-empty", []), case lists:member(NextRequest#request.id, @@ -1286,38 +1253,17 @@ handle_pipeline(#state{status = pipeline, Session#session{queue_length = %% Queue + current queue:len(Pipeline) + 1}, - insert_session(NewSession, ProfileName), - Relaxed = - (NextRequest#request.settings)#http_options.relaxed, - MFA = {httpc_response, - parse, - [State#state.max_header_size, Relaxed]}, - NewState = - State#state{pipeline = Pipeline, - request = NextRequest, - mfa = MFA, - status_line = undefined, - headers = undefined, - body = undefined}, - case Data of - <<>> -> - activate_once(Session), - {noreply, NewState}; - _ -> - %% If we already received some bytes of - %% the next response - handle_info({httpc_handler, dummy, Data}, - NewState) - end + receive_response(NextRequest, + NewSession, Data, + State#state{pipeline = Pipeline}) end end. -handle_keep_alive_queue( - #state{status = keep_alive, - session = Session, - profile_name = ProfileName, - options = #options{keep_alive_timeout = TimeOut}} = State, - Data) -> +handle_keep_alive_queue(#state{status = keep_alive, + session = Session, + profile_name = ProfileName, + options = #options{keep_alive_timeout = TimeOut}} = State, + Data) -> ?hcrd("handle keep_alive", [{profile, ProfileName}, {session, Session}, @@ -1326,25 +1272,7 @@ handle_keep_alive_queue( case queue:out(State#state.keep_alive) of {empty, _} -> ?hcrd("keep_alive queue empty", []), - %% The server may choose too terminate an idle keep_alive session - %% in this case we want to receive the close message - %% at once and not when trying to send the next - %% request. - activate_once(Session), - %% If a keep_alive session has been idle for some time is not - %% closed by the server, the client may want to close it. - NewState = activate_queue_timeout(TimeOut, State), - update_session(ProfileName, Session, #session.queue_length, 0), - %% Note mfa will be initilized when a new request - %% arrives. - {noreply, - NewState#state{request = undefined, - mfa = undefined, - status_line = undefined, - headers = undefined, - body = undefined - } - }; + handle_empty_queue(Session, ProfileName, TimeOut, State); {{value, NextRequest}, KeepAlive} -> ?hcrd("keep_alive queue non-empty", []), case lists:member(NextRequest#request.id, @@ -1355,30 +1283,61 @@ handle_keep_alive_queue( State#state{keep_alive = KeepAlive}, Data); false -> ?hcrv("next request", [{request, NextRequest}]), - Relaxed = - (NextRequest#request.settings)#http_options.relaxed, - MFA = {httpc_response, parse, - [State#state.max_header_size, Relaxed]}, - NewState = - State#state{request = NextRequest, - keep_alive = KeepAlive, - mfa = MFA, - status_line = undefined, - headers = undefined, - body = undefined}, - case Data of - <<>> -> - activate_once(Session), - {noreply, NewState}; - _ -> - %% If we already received some bytes of - %% the next response - handle_info({httpc_handler, dummy, Data}, - NewState) + #request{address = Address} = NextRequest, + case httpc_request:send(Address, Session, NextRequest) of + ok -> + receive_response(NextRequest, + Session, <<>>, + State#state{keep_alive = KeepAlive}); + {error, Reason} -> + {reply, {keep_alive_failed, Reason}, State} end end end. +handle_empty_queue(Session, ProfileName, TimeOut, State) -> + %% The server may choose too terminate an idle pipline| keep_alive session + %% in this case we want to receive the close message + %% at once and not when trying to send the next + %% request. + activate_once(Session), + %% If a pipline | keep_alive session has been idle for some time is not + %% closed by the server, the client may want to close it. + NewState = activate_queue_timeout(TimeOut, State), + update_session(ProfileName, Session, #session.queue_length, 0), + %% Note mfa will be initilized when a new request + %% arrives. + {noreply, + NewState#state{request = undefined, + mfa = undefined, + status_line = undefined, + headers = undefined, + body = undefined + } + }. + +receive_response(Request, Session, Data, State) -> + NewState = init_wait_for_response_state(Request, State), + gather_data(Data, Session, NewState). + +init_wait_for_response_state(Request, State) -> + Relaxed = + (Request#request.settings)#http_options.relaxed, + MFA = {httpc_response, parse, + [State#state.max_header_size, Relaxed]}, + State#state{request = Request, + mfa = MFA, + status_line = undefined, + headers = undefined, + body = undefined}. + +gather_data(<<>>, Session, State) -> + activate_once(Session), + {noreply, State}; +gather_data(Data, _, State) -> + %% If we already received some bytes of + %% the next response + handle_info({httpc_handler, dummy, Data}, State). case_insensitive_header(Str) when is_list(Str) -> http_util:to_lower(Str); diff --git a/lib/inets/src/http_lib/http_transport.erl b/lib/inets/src/http_lib/http_transport.erl index 5eb827032f..df58fa1b81 100644 --- a/lib/inets/src/http_lib/http_transport.erl +++ b/lib/inets/src/http_lib/http_transport.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2011. All Rights Reserved. +%% Copyright Ericsson AB 2004-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -23,7 +23,7 @@ -export([ start/1, connect/3, connect/4, - listen/2, listen/3, listen/4, + listen/4, listen/5, accept/2, accept/3, close/2, send/3, @@ -155,41 +155,41 @@ connect({essl, SslConfig}, {Host, Port}, Opts0, Timeout) -> %% reason for this to enable a HTTP-server not running as root to use %% port 80. %%------------------------------------------------------------------------- -listen(SocketType, Port) -> - listen(SocketType, undefined, Port). +listen(ip_comm = _SocketType, Addr, Port, Fd, IpFamily) -> + listen_ip_comm(Addr, Port, Fd, IpFamily); + +listen({essl, SSLConfig}, Addr, Port, Fd, IpFamily) -> + listen_ssl(Addr, Port, Fd, SSLConfig, IpFamily). -listen(ip_comm = _SocketType, Addr, Port) -> - listen_ip_comm(Addr, Port, undefined); +listen(ip_comm = _SocketType, Addr, Port, IpFamily) -> + listen_ip_comm(Addr, Port, undefined, IpFamily); %% Wrapper for backaward compatibillity -listen({ssl, SSLConfig}, Addr, Port) -> +listen({ssl, SSLConfig}, Addr, Port, IpFamily) -> ?hlrt("listen (wrapper)", [{addr, Addr}, {port, Port}, {ssl_config, SSLConfig}]), - listen({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Addr, Port); + listen({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Addr, Port, IpFamily); -listen({essl, SSLConfig}, Addr, Port) -> + +listen({essl, SSLConfig}, Addr, Port, IpFamily) -> ?hlrt("listen (essl)", [{addr, Addr}, {port, Port}, {ssl_config, SSLConfig}]), - listen_ssl(Addr, Port, [{ssl_imp, new}, {reuseaddr, true} | SSLConfig]). - + listen_ssl(Addr, Port, undefined, SSLConfig, IpFamily). -listen(ip_comm, Addr, Port, Fd) -> - listen_ip_comm(Addr, Port, Fd). - -listen_ip_comm(Addr, Port, Fd) -> - case (catch do_listen_ip_comm(Addr, Port, Fd)) of +listen_ip_comm(Addr, Port, Fd, IpFamily) -> + case (catch do_listen_ip_comm(Addr, Port, Fd, IpFamily)) of {'EXIT', Reason} -> {error, {exit, Reason}}; Else -> Else end. -do_listen_ip_comm(Addr, Port, Fd) -> - {NewPort, Opts, IpFamily} = get_socket_info(Addr, Port, Fd), +do_listen_ip_comm(Addr, Port, Fd, IpFamily) -> + {NewPort, Opts} = get_socket_info(Addr, Port, Fd), case IpFamily of inet6fb4 -> Opts2 = [inet6 | Opts], @@ -222,10 +222,9 @@ do_listen_ip_comm(Addr, Port, Fd) -> end. -listen_ssl(Addr, Port, Opts0) -> - IpFamily = ipfamily_default(Addr, Port), - BaseOpts = [{backlog, 128}, {reuseaddr, true} | Opts0], - Opts = sock_opts(Addr, BaseOpts), +listen_ssl(Addr, Port, Fd, Opts0, IpFamily) -> + {NewPort, SockOpt} = get_socket_info(Addr, Port, Fd), + Opts = SockOpt ++ Opts0, case IpFamily of inet6fb4 -> Opts2 = [inet6 | Opts], @@ -236,13 +235,13 @@ listen_ssl(Addr, Port, Opts0) -> Opts3 = [inet | Opts], ?hlrt("ipv6 listen failed - try ipv4 instead", [{reason, Reason}, {opts, Opts3}]), - ssl:listen(Port, Opts3); + ssl:listen(NewPort, Opts3); {'EXIT', Reason} -> Opts3 = [inet | Opts], ?hlrt("ipv6 listen exit - try ipv4 instead", [{reason, Reason}, {opts, Opts3}]), - ssl:listen(Port, Opts3); + ssl:listen(NewPort, Opts3); Other -> ?hlrt("ipv6 listen done", [{other, Other}]), @@ -252,61 +251,21 @@ listen_ssl(Addr, Port, Opts0) -> _ -> Opts2 = [IpFamily | Opts], ?hlrt("listen", [{opts, Opts2}]), - ssl:listen(Port, Opts2) + ssl:listen(NewPort, Opts2) end. -ipfamily_default(Addr, Port) -> - httpd_conf:lookup(Addr, Port, ipfamily, inet6fb4). -get_socket_info(Addr, Port, Fd0) -> +get_socket_info(Addr, Port, Fd) -> BaseOpts = [{backlog, 128}, {reuseaddr, true}], - IpFamilyDefault = ipfamily_default(Addr, Port), %% The presence of a file descriptor takes precedence - case get_fd(Port, Fd0, IpFamilyDefault) of - {Fd, IpFamily} -> - {0, sock_opts(Addr, [{fd, Fd} | BaseOpts]), IpFamily}; + case Fd of undefined -> - {Port, sock_opts(Addr, BaseOpts), IpFamilyDefault} + {Port, sock_opts(Addr, BaseOpts)}; + Fd -> + {0, sock_opts(Addr, [{fd, Fd} | BaseOpts])} end. -get_fd(Port, undefined = _Fd, IpFamilyDefault) -> - FdKey = list_to_atom("httpd_" ++ integer_to_list(Port)), - case init:get_argument(FdKey) of - {ok, [[Value]]} -> - case string:tokens(Value, [$|]) of - [FdStr, IpFamilyStr] -> - {fd_of(FdStr), ip_family_of(IpFamilyStr)}; - [FdStr] -> - {fd_of(FdStr), IpFamilyDefault}; - _ -> - throw({error, {bad_descriptor, Value}}) - end; - error -> - undefined - end; -get_fd(_Port, Fd, IpFamilyDefault) -> - {Fd, IpFamilyDefault}. - - -fd_of(FdStr) -> - case (catch list_to_integer(FdStr)) of - Fd when is_integer(Fd) -> - Fd; - _ -> - throw({error, {bad_descriptor, FdStr}}) - end. - -ip_family_of(IpFamilyStr) -> - IpFamily = list_to_atom(IpFamilyStr), - case lists:member(IpFamily, [inet, inet6, inet6fb4]) of - true -> - IpFamily; - false -> - throw({error, {bad_ipfamily, IpFamilyStr}}) - end. - - %%------------------------------------------------------------------------- %% accept(SocketType, ListenSocket) -> {ok, Socket} | {error, Reason} %% accept(SocketType, ListenSocket, Timeout) -> ok | {error, Reason} diff --git a/lib/inets/src/http_server/httpd_acceptor.erl b/lib/inets/src/http_server/httpd_acceptor.erl index 08ee9ee0d0..1bffcc1f12 100644 --- a/lib/inets/src/http_server/httpd_acceptor.erl +++ b/lib/inets/src/http_server/httpd_acceptor.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2011. All Rights Reserved. +%% Copyright Ericsson AB 2001-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -24,10 +24,10 @@ -include("inets_internal.hrl"). %% Internal application API --export([start_link/5, start_link/6]). +-export([start_link/6, start_link/7]). %% Other exports (for spawn's etc.) --export([acceptor_init/6, acceptor_init/7, acceptor_loop/5]). +-export([acceptor_init/7, acceptor_init/8, acceptor_loop/6]). %% %% External API @@ -35,27 +35,27 @@ %% start_link -start_link(Manager, SocketType, Addr, Port, ConfigDb, AcceptTimeout) -> +start_link(Manager, SocketType, Addr, Port, IpFamily, ConfigDb, AcceptTimeout) -> ?hdrd("start link", [{manager, Manager}, {socket_type, SocketType}, {address, Addr}, {port, Port}, {timeout, AcceptTimeout}]), - Args = [self(), Manager, SocketType, Addr, Port, ConfigDb, AcceptTimeout], + Args = [self(), Manager, SocketType, Addr, Port, IpFamily, ConfigDb, AcceptTimeout], proc_lib:start_link(?MODULE, acceptor_init, Args). -start_link(Manager, SocketType, ListenSocket, ConfigDb, AcceptTimeout) -> +start_link(Manager, SocketType, ListenSocket, IpFamily, ConfigDb, AcceptTimeout) -> ?hdrd("start link", [{manager, Manager}, {socket_type, SocketType}, {listen_socket, ListenSocket}, {timeout, AcceptTimeout}]), - Args = [self(), Manager, SocketType, ListenSocket, + Args = [self(), Manager, SocketType, ListenSocket, IpFamily, ConfigDb, AcceptTimeout], proc_lib:start_link(?MODULE, acceptor_init, Args). -acceptor_init(Parent, Manager, SocketType, {ListenOwner, ListenSocket}, +acceptor_init(Parent, Manager, SocketType, {ListenOwner, ListenSocket}, IpFamily, ConfigDb, AcceptTimeout) -> ?hdrd("acceptor init", [{parent, Parent}, @@ -66,9 +66,9 @@ acceptor_init(Parent, Manager, SocketType, {ListenOwner, ListenSocket}, {timeout, AcceptTimeout}]), link(ListenOwner), proc_lib:init_ack(Parent, {ok, self()}), - acceptor_loop(Manager, SocketType, ListenSocket, ConfigDb, AcceptTimeout). + acceptor_loop(Manager, SocketType, ListenSocket, IpFamily, ConfigDb, AcceptTimeout). -acceptor_init(Parent, Manager, SocketType, Addr, Port, +acceptor_init(Parent, Manager, SocketType, Addr, Port, IpFamily, ConfigDb, AcceptTimeout) -> ?hdrd("acceptor init", [{parent, Parent}, @@ -77,20 +77,20 @@ acceptor_init(Parent, Manager, SocketType, Addr, Port, {address, Addr}, {port, Port}, {timeout, AcceptTimeout}]), - case (catch do_init(SocketType, Addr, Port)) of + case (catch do_init(SocketType, Addr, Port, IpFamily)) of {ok, ListenSocket} -> proc_lib:init_ack(Parent, {ok, self()}), acceptor_loop(Manager, SocketType, - ListenSocket, ConfigDb, AcceptTimeout); + ListenSocket, IpFamily,ConfigDb, AcceptTimeout); Error -> proc_lib:init_ack(Parent, Error), error end. -do_init(SocketType, Addr, Port) -> +do_init(SocketType, Addr, Port, IpFamily) -> ?hdrt("do init", []), do_socket_start(SocketType), - ListenSocket = do_socket_listen(SocketType, Addr, Port), + ListenSocket = do_socket_listen(SocketType, Addr, Port, IpFamily), {ok, ListenSocket}. @@ -105,9 +105,9 @@ do_socket_start(SocketType) -> end. -do_socket_listen(SocketType, Addr, Port) -> +do_socket_listen(SocketType, Addr, Port, IpFamily) -> ?hdrt("do socket listen", []), - case http_transport:listen(SocketType, Addr, Port) of + case http_transport:listen(SocketType, Addr, Port, IpFamily) of {ok, ListenSocket} -> ListenSocket; {error, Reason} -> @@ -121,7 +121,7 @@ do_socket_listen(SocketType, Addr, Port) -> %% acceptor -acceptor_loop(Manager, SocketType, ListenSocket, ConfigDb, AcceptTimeout) -> +acceptor_loop(Manager, SocketType, ListenSocket, IpFamily, ConfigDb, AcceptTimeout) -> ?hdrd("awaiting accept", [{manager, Manager}, {socket_type, SocketType}, @@ -133,12 +133,12 @@ acceptor_loop(Manager, SocketType, ListenSocket, ConfigDb, AcceptTimeout) -> handle_connection(Manager, ConfigDb, AcceptTimeout, SocketType, Socket), ?MODULE:acceptor_loop(Manager, SocketType, - ListenSocket, ConfigDb,AcceptTimeout); + ListenSocket, IpFamily, ConfigDb,AcceptTimeout); {error, Reason} -> ?hdri("accept failed", [{reason, Reason}]), handle_error(Reason, ConfigDb), ?MODULE:acceptor_loop(Manager, SocketType, ListenSocket, - ConfigDb, AcceptTimeout); + IpFamily, ConfigDb, AcceptTimeout); {'EXIT', Reason} -> ?hdri("accept exited", [{reason, Reason}]), ReasonString = diff --git a/lib/inets/src/http_server/httpd_acceptor_sup.erl b/lib/inets/src/http_server/httpd_acceptor_sup.erl index 8b1e4b6c4f..df837b5a24 100644 --- a/lib/inets/src/http_server/httpd_acceptor_sup.erl +++ b/lib/inets/src/http_server/httpd_acceptor_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -27,7 +27,7 @@ -behaviour(supervisor). %% API --export([start_link/2, start_acceptor/5, start_acceptor/6, stop_acceptor/2]). +-export([start_link/2, start_acceptor/6, start_acceptor/7, stop_acceptor/2]). %% Supervisor callback -export([init/1]). @@ -43,11 +43,11 @@ start_link(Addr, Port) -> %% Function: [start|stop]_acceptor/5 %% Description: Starts/stops an [auth | security] worker (child) process %%---------------------------------------------------------------------- -start_acceptor(SocketType, Addr, Port, ConfigDb, AcceptTimeout) -> - start_worker(httpd_acceptor, SocketType, Addr, Port, +start_acceptor(SocketType, Addr, Port, IpFamily, ConfigDb, AcceptTimeout) -> + start_worker(httpd_acceptor, SocketType, Addr, Port, IpFamily, ConfigDb, AcceptTimeout, self(), []). -start_acceptor(SocketType, Addr, Port, ConfigDb, AcceptTimeout, ListenSocket) -> - start_worker(httpd_acceptor, SocketType, Addr, Port, +start_acceptor(SocketType, Addr, Port, IpFamily, ConfigDb, AcceptTimeout, ListenSocket) -> + start_worker(httpd_acceptor, SocketType, Addr, Port, IpFamily, ConfigDb, AcceptTimeout, ListenSocket, self(), []). @@ -69,18 +69,18 @@ init(_) -> make_name(Addr,Port) -> httpd_util:make_name("httpd_acc_sup", Addr, Port). -start_worker(M, SocketType, Addr, Port, ConfigDB, AcceptTimeout, Manager, Modules) -> +start_worker(M, SocketType, Addr, Port, IpFamily, ConfigDB, AcceptTimeout, Manager, Modules) -> SupName = make_name(Addr, Port), - Args = [Manager, SocketType, Addr, Port, ConfigDB, AcceptTimeout], + Args = [Manager, SocketType, Addr, Port, IpFamily, ConfigDB, AcceptTimeout], Spec = {{M, Addr, Port}, {M, start_link, Args}, permanent, timer:seconds(1), worker, [M] ++ Modules}, supervisor:start_child(SupName, Spec). -start_worker(M, SocketType, Addr, Port, ConfigDB, AcceptTimeout, ListenSocket, +start_worker(M, SocketType, Addr, Port, IpFamily, ConfigDB, AcceptTimeout, ListenSocket, Manager, Modules) -> SupName = make_name(Addr, Port), - Args = [Manager, SocketType, ListenSocket, ConfigDB, AcceptTimeout], + Args = [Manager, SocketType, ListenSocket, IpFamily, ConfigDB, AcceptTimeout], Spec = {{M, Addr, Port}, {M, start_link, Args}, permanent, timer:seconds(1), worker, [M] ++ Modules}, diff --git a/lib/inets/src/http_server/httpd_conf.erl b/lib/inets/src/http_server/httpd_conf.erl index a97bbd9b25..d45f3c0048 100644 --- a/lib/inets/src/http_server/httpd_conf.erl +++ b/lib/inets/src/http_server/httpd_conf.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -582,11 +582,17 @@ validate_config_params([{server_tokens, {private, Value}} | Rest]) validate_config_params([{server_tokens, Value} | _]) -> throw({server_tokens, Value}); +validate_config_params([{socket_type, ip_comm} | Rest]) -> + validate_config_params(Rest); + validate_config_params([{socket_type, Value} | Rest]) - when (Value =:= ip_comm) orelse - (Value =:= ssl) orelse - (Value =:= essl) -> + when Value == ssl; Value == essl -> validate_config_params(Rest); + +validate_config_params([{socket_type, {Value, _}} | Rest]) + when Value == essl orelse Value == ssl -> + validate_config_params(Rest); + validate_config_params([{socket_type, Value} | _]) -> throw({socket_type, Value}); @@ -916,6 +922,8 @@ lookup_socket_type(ConfigDB) -> case httpd_util:lookup(ConfigDB, socket_type, ip_comm) of ip_comm -> ip_comm; + {Tag, Conf} -> + {Tag, Conf}; SSL when (SSL =:= ssl) orelse (SSL =:= essl) -> SSLTag = if diff --git a/lib/inets/src/http_server/httpd_manager.erl b/lib/inets/src/http_server/httpd_manager.erl index b44bc77c41..672a70a394 100644 --- a/lib/inets/src/http_server/httpd_manager.erl +++ b/lib/inets/src/http_server/httpd_manager.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2010. All Rights Reserved. +%% Copyright Ericsson AB 2000-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -264,11 +264,12 @@ init([ConfigFile, ConfigList, AcceptTimeout, Addr, Port, ListenInfo]) -> end. do_init(ConfigFile, ConfigList, AcceptTimeout, Addr, Port) -> + IpFamily = proplists:get_value(ipfamily, ConfigList, inet6fb4), NewConfigFile = proplists:get_value(file, ConfigList, ConfigFile), ConfigDB = do_initial_store(ConfigList), SocketType = httpd_conf:lookup_socket_type(ConfigDB), case httpd_acceptor_sup:start_acceptor(SocketType, Addr, - Port, ConfigDB, AcceptTimeout) of + Port, IpFamily, ConfigDB, AcceptTimeout) of {ok, _Pid} -> Status = [{max_conn, 0}, {last_heavy_load, never}, @@ -284,11 +285,12 @@ do_init(ConfigFile, ConfigList, AcceptTimeout, Addr, Port) -> end. do_init(ConfigFile, ConfigList, AcceptTimeout, Addr, Port, ListenInfo) -> + IpFamily = proplists:get_value(ipfamily, ConfigList, inet6fb4), NewConfigFile = proplists:get_value(file, ConfigList, ConfigFile), ConfigDB = do_initial_store(ConfigList), SocketType = httpd_conf:lookup_socket_type(ConfigDB), case httpd_acceptor_sup:start_acceptor(SocketType, Addr, - Port, ConfigDB, + Port, IpFamily, ConfigDB, AcceptTimeout, ListenInfo) of {ok, _Pid} -> Status = [{max_conn,0}, {last_heavy_load,never}, diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index 2dedb088e4..6b6532266b 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -35,8 +35,7 @@ generate_and_send_response(#mod{init_data = #init_data{peername = {_,"unknown"}}}) -> ok; generate_and_send_response(#mod{config_db = ConfigDB} = ModData) -> - Modules = httpd_util:lookup(ConfigDB,modules, - [mod_get, mod_head, mod_log]), + Modules = httpd_util:lookup(ConfigDB,modules, ?DEFAULT_MODS), case traverse_modules(ModData, Modules) of done -> ok; @@ -71,7 +70,6 @@ traverse_modules(ModData,[Module|Rest]) -> ?hdrd("traverse modules", [{callback_module, Module}]), case (catch apply(Module, do, [ModData])) of {'EXIT', Reason} -> - ?hdrd("traverse modules - exit", [{reason, Reason}]), String = lists:flatten( io_lib:format("traverse exit from apply: ~p:do => ~n~p", diff --git a/lib/inets/src/http_server/httpd_sup.erl b/lib/inets/src/http_server/httpd_sup.erl index 8f3e8f9500..3b1e16cf78 100644 --- a/lib/inets/src/http_server/httpd_sup.erl +++ b/lib/inets/src/http_server/httpd_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2012. All Rights Reserved. +%% Copyright Ericsson AB 2004-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -196,7 +196,8 @@ httpd_child_spec(ConfigFile, AcceptTimeoutDef, DebugDef) -> end. httpd_child_spec(Config, AcceptTimeout, Debug, Addr, Port) -> - case (Port =:= 0) orelse proplists:is_defined(fd, Config) of + Fd = proplists:get_value(fd, Config, undefined), + case Port == 0 orelse Fd =/= undefined of true -> httpd_child_spec_listen(Config, AcceptTimeout, Debug, Addr, Port); false -> @@ -242,21 +243,27 @@ error_msg(F, A) -> error_logger:error_msg(F ++ "~n", A). listen(Address, Port, Config) -> - SocketType = proplists:get_value(socket_type, Config, ip_comm), - case http_transport:start(SocketType) of - ok -> - Fd = proplists:get_value(fd, Config), - case http_transport:listen(SocketType, Address, Port, Fd) of - {ok, ListenSocket} -> - NewConfig = proplists:delete(port, Config), - {ok, NewPort} = inet:port(ListenSocket), - {NewPort, [{port, NewPort} | NewConfig], ListenSocket}; + try socket_type(Config) of + SocketType -> + case http_transport:start(SocketType) of + ok -> + Fd = proplists:get_value(fd, Config), + IpFamily = proplists:get_value(ipfamily, Config, inet6fb4), + case http_transport:listen(SocketType, Address, Port, Fd, IpFamily) of + {ok, ListenSocket} -> + NewConfig = proplists:delete(port, Config), + {NewPort, _} = http_transport:sockname(SocketType, ListenSocket), + {NewPort, [{port, NewPort} | NewConfig], ListenSocket}; + {error, Reason} -> + {error, {listen, Reason}} + end; {error, Reason} -> - {error, {listen, Reason}} - end; - {error, Reason} -> + {error, {socket_start_failed, Reason}} + end + catch + _:Reason -> {error, {socket_start_failed, Reason}} - end. + end. start_listen(Address, Port, Config) -> Pid = listen_owner(Address, Port, Config), @@ -280,7 +287,82 @@ listen_loop() -> ok end. +socket_type(Config) -> + SocketType = proplists:get_value(socket_type, Config, ip_comm), + socket_type(SocketType, Config). + +socket_type(ip_comm = SocketType, _) -> + SocketType; +socket_type({essl, _} = SocketType, _) -> + SocketType; +socket_type(_, Config) -> + {essl, ssl_config(Config)}. + +%%% Backwards compatibility +ssl_config(Config) -> + ssl_certificate_key_file(Config) ++ + ssl_verify_client(Config) ++ + ssl_ciphers(Config) ++ + ssl_password(Config) ++ + ssl_verify_depth(Config) ++ + ssl_ca_certificate_file(Config). + +ssl_certificate_key_file(Config) -> + case proplists:get_value(ssl_certificate_key_file, Config) of + undefined -> + []; + SSLCertificateKeyFile -> + [{keyfile,SSLCertificateKeyFile}] + end. +ssl_verify_client(Config) -> + case proplists:get_value(ssl_verify_client, Config) of + undefined -> + []; + SSLVerifyClient -> + [{verify,SSLVerifyClient}] + end. +ssl_ciphers(Config) -> + case proplists:get_value(ssl_ciphers, Config) of + undefined -> + []; + Ciphers -> + [{ciphers, Ciphers}] + end. +ssl_password(Config) -> + case proplists:get_value(ssl_password_callback_module, Config) of + undefined -> + []; + Module -> + case proplists:get_value(ssl_password_callback_function, Config) of + undefined -> + []; + Function -> + Args = case proplists:get_value(ssl_password_callback_arguments, Config) of + undefined -> + []; + Arguments -> + [Arguments] + end, + Password = apply(Module, Function, Args), + [{password, Password}] + end + end. +ssl_verify_depth(Config) -> + case proplists:get_value(ssl_verify_client_depth, Config) of + undefined -> + []; + Depth -> + [{depth, Depth}] + end. + +ssl_ca_certificate_file(Config) -> + case proplists:get_value(ssl_ca_certificate_file, Config) of + undefined -> + []; + File -> + [{cacertfile, File}] + end. diff --git a/lib/inets/test/Makefile b/lib/inets/test/Makefile index 0ca99e8692..dfa86906fd 100644 --- a/lib/inets/test/Makefile +++ b/lib/inets/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -184,7 +184,8 @@ MODULES = \ inets_app_test \ inets_appup_test \ tftp_test_lib \ - tftp_SUITE + tftp_SUITE \ + uri_SUITE EBIN = . diff --git a/lib/inets/test/ftp_format_SUITE.erl b/lib/inets/test/ftp_format_SUITE.erl index cbc1b04bbb..16e5cdb4bc 100644 --- a/lib/inets/test/ftp_format_SUITE.erl +++ b/lib/inets/test/ftp_format_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2011. All Rights Reserved. +%% Copyright Ericsson AB 2005-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -273,7 +273,7 @@ ftp_other_status_codes(Config) when is_list(Config) -> %% 4XX {trans_neg_compl, _ } = ftp_response:interpret("421 Foobar\r\n"), {trans_neg_compl, _ } = ftp_response:interpret("426 Foobar\r\n"), - {trans_neg_compl, _ } = ftp_response:interpret("450 Foobar\r\n"), + {enofile, _ } = ftp_response:interpret("450 Foobar\r\n"), {trans_neg_compl, _ } = ftp_response:interpret("451 Foobar\r\n"), {etnospc, _ } = ftp_response:interpret("452 Foobar\r\n"), diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 5a3bdaefcf..8df5964193 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -19,2656 +19,919 @@ %% %% -%% ts:run(inets, httpc_SUITE, [batch]). -%% +%% ct:run("../inets_test", httpc_SUITE). +%% -module(httpc_SUITE). --include_lib("common_test/include/ct.hrl"). --include("test_server_line.hrl"). - -include_lib("kernel/include/file.hrl"). +-include_lib("common_test/include/ct.hrl"). -include("inets_test_lib.hrl"). %% Note: This directive should only be used in test suites. -compile(export_all). -%% Test server specific exports --define(IP_PORT, 8998). --define(SSL_PORT, 8999). +-define(URL_START, "http://"). +-define(TLS_URL_START, "https://"). -define(NOT_IN_USE_PORT, 8997). --define(LOCAL_HOST, {127,0,0,1}). --define(IPV6_LOCAL_HOST, "0:0:0:0:0:0:0:1"). --define(URL_START, "http://localhost:"). --define(SSL_URL_START, "https://localhost:"). --define(CR, $\r). -define(LF, $\n). -define(HTTP_MAX_HEADER_SIZE, 10240). - - +-record(sslsocket, {fd = nil, pid = nil}). %%-------------------------------------------------------------------- -%% all(Arg) -> [Doc] | [Case] | {skip, Comment} -%% Arg - doc | suite -%% Doc - string() -%% Case - atom() -%% Name of a test case function. -%% Comment - string() -%% Description: Returns documentation/test cases in this test suite -%% or a skip tuple if the platform is not supported. +%% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- +suite() -> + [{ct_hooks,[ts_install_cth]}]. -suite() -> [{ct_hooks,[ts_install_cth]}]. - -all() -> +all() -> [ - http_options, - http_head, - http_get, - http_post, - http_post_streaming, - http_dummy_pipe, - http_inets_pipe, - http_trace, - http_async, - http_save_to_file, - http_save_to_file_async, - http_headers, - http_headers_dummy, - http_bad_response, - http_redirect, - http_redirect_loop, - http_internal_server_error, - http_userinfo, http_cookie, - http_server_does_not_exist, - http_invalid_http, - http_emulate_lower_versions, - http_relaxed, - page_does_not_exist, - parse_url, - options, - headers_as_is, - selecting_session, - {group, ssl}, - {group, stream}, - {group, ipv6}, - {group, tickets}, - initial_server_connect + {group, http}, + {group, sim_http}, + {group, https}, + {group, sim_https}, + {group, misc} ]. -groups() -> +groups() -> [ - {ssl, [], [ssl_head, - essl_head, - ssl_get, - essl_get, - ssl_trace, - essl_trace]}, - {stream, [], [http_stream, - http_stream_once]}, - {tickets, [], [hexed_query_otp_6191, - empty_body_otp_6243, - empty_response_header_otp_6830, - transfer_encoding_otp_6807, - no_content_204_otp_6982, - missing_CR_otp_7304, - {group, otp_7883}, - {group, otp_8154}, - {group, otp_8106}, - otp_8056, - otp_8352, - otp_8371, - otp_8739]}, - {otp_7883, [], [otp_7883_1, - otp_7883_2]}, - {otp_8154, [], [otp_8154_1]}, - {otp_8106, [], [otp_8106_pid, - otp_8106_fun, - otp_8106_mfa]}, - {ipv6, [], [ipv6_ipcomm, ipv6_essl]} + {http, [], real_requests()}, + {sim_http, [], only_simulated()}, + {https, [], real_requests()}, + {sim_https, [], only_simulated()}, + {misc, [], misc()} ]. +real_requests()-> + [ + head, + get, + post, + post_stream, + async, + pipeline, + persistent_connection, + save_to_file, + save_to_file_async, + headers_as_is, + page_does_not_exist, + emulate_lower_versions, + headers, + headers_as_is, + empty_body, + stream, + stream_to_pid, + stream_through_fun, + stream_through_mfa, + streaming_error, + inet_opts + ]. -init_per_group(ipv6 = _GroupName, Config) -> - case inets_test_lib:has_ipv6_support() of - {ok, _} -> - Config; - _ -> - {skip, "Host does not support IPv6"} - end; -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, Config) -> - Config. +only_simulated() -> + [ + cookie, + trace, + stream_once, + no_content_204, + tolerate_missing_CR, + userinfo, + bad_response, + internal_server_error, + invalid_http, + headers_dummy, + empty_response_header, + remote_socket_close, + remote_socket_close_async, + transfer_encoding, + redirect_loop, + redirect_moved_permanently, + redirect_multiple_choises, + redirect_found, + redirect_see_other, + redirect_temporary_redirect, + port_in_host_header, + relaxed + ]. +misc() -> + [ + server_does_not_exist, + timeout_memory_leak, + wait_for_whole_response + ]. %%-------------------------------------------------------------------- -%% Function: init_per_suite(Config) -> Config -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Initiation before the whole suite -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%%-------------------------------------------------------------------- -init_per_suite(Config) -> - - ?PRINT_SYSTEM_INFO([]), +init_per_suite(Config) -> PrivDir = ?config(priv_dir, Config), DataDir = ?config(data_dir, Config), + inets_test_lib:start_apps([inets]), ServerRoot = filename:join(PrivDir, "server_root"), DocRoot = filename:join(ServerRoot, "htdocs"), - IpConfFile = integer_to_list(?IP_PORT) ++ ".conf", - SslConfFile = integer_to_list(?SSL_PORT) ++ ".conf", - setup_server_dirs(ServerRoot, DocRoot, DataDir), - create_config(IpConfFile, ip_comm, ?IP_PORT, PrivDir, ServerRoot, - DocRoot, DataDir), - create_config(SslConfFile, ssl, ?SSL_PORT, PrivDir, ServerRoot, - DocRoot, DataDir), - - Cgi = case test_server:os_type() of - {win32, _} -> - filename:join([ServerRoot, "cgi-bin", "cgi_echo.exe"]); - _ -> - filename:join([ServerRoot, "cgi-bin", "cgi_echo"]) - end, - - {ok, FileInfo} = file:read_file_info(Cgi), - ok = file:write_file_info(Cgi, FileInfo#file_info{mode = 8#00755}), - - [{has_ipv6_support, inets_test_lib:has_ipv6_support()}, - {server_root, ServerRoot}, - {doc_root, DocRoot}, - {local_port, ?IP_PORT}, - {local_ssl_port, ?SSL_PORT} | Config]. + [{server_root, ServerRoot}, {doc_root, DocRoot} | Config]. - -%%-------------------------------------------------------------------- -%% Function: end_per_suite(Config) -> _ -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Cleanup after the whole suite -%%-------------------------------------------------------------------- end_per_suite(Config) -> - PrivDir = ?config(priv_dir, Config), + inets_test_lib:stop_apps([inets]), + PrivDir = ?config(priv_dir, Config), inets_test_lib:del_dirs(PrivDir), - application:stop(inets), - application:stop(ssl), ok. - %%-------------------------------------------------------------------- -%% Function: init_per_testcase(Case, Config) -> Config -%% Case - atom() -%% Name of the test case that is about to be run. -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% -%% Description: Initiation before each test case -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. +init_per_group(misc = Group, Config) -> + start_apps(Group), + Inet = inet_version(), + ok = httpc:set_options([{ipfamily, Inet}]), + Config; + +init_per_group(Group, Config0) -> + start_apps(Group), + Config = proplists:delete(port, Config0), + Port = server_start(Group, server_config(Group, Config)), + [{port, Port} | Config]. + +end_per_group(_, _Config) -> + ok. + %%-------------------------------------------------------------------- +init_per_testcase(pipeline, Config) -> + inets:start(httpc, [{profile, pipeline}]), + httpc:set_options([{pipeline_timeout, 50000}, + {max_pipeline_length, 3}], pipeline), -init_per_testcase(otp_8154_1 = Case, Config) -> - init_per_testcase(Case, 5, Config); - -init_per_testcase(initial_server_connect = Case, Config) -> - %% Try to check if crypto actually exist or not, - %% this test case does not work unless it does - try - begin - ?ENSURE_STARTED([crypto, public_key, ssl]), - inets:start(), - Config - end - catch - throw:{error, {failed_starting, App, ActualError}} -> - tsp("init_per_testcase(~w) -> failed starting ~w: " - "~n ~p", [Case, App, ActualError]), - SkipString = - "Could not start " ++ atom_to_list(App), - skip(SkipString); - _:X -> - SkipString = - lists:flatten( - io_lib:format("Failed starting apps: ~p", [X])), - skip(SkipString) - end; + Config; +init_per_testcase(persistent_connection, Config) -> + inets:start(httpc, [{profile, persistent}]), + httpc:set_options([{keep_alive_timeout, 50000}, + {max_keep_alive_length, 3}], persistent_connection), -init_per_testcase(Case, Config) -> - init_per_testcase(Case, 2, Config). + Config; -init_per_testcase(Case, Timeout, Config) -> - io:format(user, - "~n~n*** INIT ~w:~w[~w] ***" - "~n~n", [?MODULE, Case, Timeout]), +init_per_testcase(_Case, Config) -> + Config. - PrivDir = ?config(priv_dir, Config), - application:stop(inets), - Dog = test_server:timetrap(inets_test_lib:minutes(Timeout)), - TmpConfig = lists:keydelete(watchdog, 1, Config), - IpConfFile = integer_to_list(?IP_PORT) ++ ".conf", - SslConfFile = integer_to_list(?SSL_PORT) ++ ".conf", - - %% inets:enable_trace(max, io, httpd), - %% inets:enable_trace(max, io, httpc), - %% inets:enable_trace(max, io, all), - - NewConfig = - case atom_to_list(Case) of - [$s, $s, $l | _] -> - ?ENSURE_STARTED([crypto, public_key, ssl]), - init_per_testcase_ssl(ssl, PrivDir, SslConfFile, - [{watchdog, Dog} | TmpConfig]); - - [$e, $s, $s, $l | _] -> - ?ENSURE_STARTED([crypto, public_key, ssl]), - init_per_testcase_ssl(essl, PrivDir, SslConfFile, - [{watchdog, Dog} | TmpConfig]); - - "ipv6_" ++ _Rest -> - %% Ensure needed apps (crypto, public_key and ssl) are started - try ?ENSURE_STARTED([crypto, public_key, ssl]) of - ok -> - Profile = ipv6, - %% A stand-alone profile is represented by a pid() - {ok, ProfilePid} = - inets:start(httpc, - [{profile, Profile}, - {data_dir, PrivDir}], stand_alone), - ok = httpc:set_options([{ipfamily, inet6}], - ProfilePid), - tsp("httpc profile pid: ~p", [ProfilePid]), - [{watchdog, Dog}, {profile, ProfilePid}| TmpConfig] - catch - throw:{error, {failed_starting, App, ActualError}} -> - tsp("init_per_testcase(~w) -> failed starting ~w: " - "~n ~p", [Case, App, ActualError]), - SkipString = - "Could not start " ++ atom_to_list(App), - skip(SkipString); - _:X -> - SkipString = - lists:flatten( - io_lib:format("Failed starting apps: ~p", [X])), - skip(SkipString) - end; - - _ -> - %% Try inet6fb4 on windows... - %% No need? Since it is set above? - - %% tsp("init_per_testcase -> allways try IPv6 on windows"), - %% ?RUN_ON_WINDOWS( - %% fun() -> - %% tsp("init_per_testcase:set_options_fun -> " - %% "set-option ipfamily to inet6fb4"), - %% Res = httpc:set_options([{ipfamily, inet6fb4}]), - %% tsp("init_per_testcase:set_options_fun -> " - %% "~n Res: ~p", [Res]), - %% Res - %% end), - - TmpConfig2 = lists:keydelete(local_server, 1, TmpConfig), - %% Will start inets - tsp("init_per_testcase -> try start server"), - Server = start_http_server(PrivDir, IpConfFile), - [{watchdog, Dog}, {local_server, Server} | TmpConfig2] - end, - - %% <IPv6> - %% Set default ipfamily to the same as the main server has by default - %% This makes the client try w/ ipv6 before falling back to ipv4, - %% as that is what the server is configured to do. - %% Note that this is required for the tests to run on *BSD w/ ipv6 enabled - %% as well as on Windows. The Linux behaviour of allowing ipv4 connects - %% to ipv6 sockets is not required or even encouraged. - - tsp("init_per_testcase -> Options before ipfamily set: ~n~p", - [httpc:get_options(all)]), - ok = httpc:set_options([{ipfamily, inet6fb4}]), - tsp("init_per_testcase -> Options after ipfamily set: ~n~p", - [httpc:get_options(all)]), - - %% Note that the IPv6 test case(s) *must* use inet6, - %% so this value will be overwritten (see "ipv6_" below). - %% </IPv6> - - %% inets:enable_trace(max, io, all), - %% snmp:set_trace([gen_tcp]), - tsp("init_per_testcase(~w) -> done when" - "~n NewConfig: ~p" - "~n~n", [Case, NewConfig]), - NewConfig. - - -init_per_testcase_ssl(Tag, PrivDir, SslConfFile, Config) -> - tsp("init_per_testcase_ssl(~w) -> stop ssl", [Tag]), - application:stop(ssl), - Config2 = lists:keydelete(local_ssl_server, 1, Config), - %% Will start inets - tsp("init_per_testcase_ssl(~w) -> try start http server (including inets)", - [Tag]), - Server = inets_test_lib:start_http_server( - filename:join(PrivDir, SslConfFile), Tag), - tsp("init_per_testcase(~w) -> Server: ~p", [Tag, Server]), - [{local_ssl_server, Server} | Config2]. - -start_http_server(ConfDir, ConfFile) -> - inets_test_lib:start_http_server( filename:join(ConfDir, ConfFile) ). +end_per_testcase(pipeline, _Config) -> + inets:stop(httpc, pipeline); +end_per_testcase(persistent_connection, _Config) -> + inets:stop(httpc, persistent); +end_per_testcase(_Case, _Config) -> + ok. %%-------------------------------------------------------------------- -%% Function: end_per_testcase(Case, Config) -> _ -%% Case - atom() -%% Name of the test case that is about to be run. -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Cleanup after each test case +%% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -end_per_testcase(http_save_to_file = Case, Config) -> - io:format(user, "~n~n*** END ~w:~w ***~n~n", - [?MODULE, Case]), - PrivDir = ?config(priv_dir, Config), - FullPath = filename:join(PrivDir, "dummy.html"), - file:delete(FullPath), - finish(Config); - -end_per_testcase(Case, Config) -> - io:format(user, "~n~n*** END ~w:~w ***~n~n", - [?MODULE, Case]), - case atom_to_list(Case) of - "ipv6_" ++ _Rest -> - tsp("end_per_testcase(~w) -> stop ssl", [Case]), - application:stop(ssl), - tsp("end_per_testcase(~w) -> stop public_key", [Case]), - application:stop(public_key), - tsp("end_per_testcase(~w) -> stop crypto", [Case]), - application:stop(crypto), - ProfilePid = ?config(profile, Config), - tsp("end_per_testcase(~w) -> stop httpc profile (~p)", - [Case, ProfilePid]), - unlink(ProfilePid), - inets:stop(stand_alone, ProfilePid), - tsp("end_per_testcase(~w) -> httpc profile (~p) stopped", - [Case, ProfilePid]), - ok; - _ -> - ok - end, - finish(Config). -finish(Config) -> - Dog = ?config(watchdog, Config), - case Dog of - undefined -> - ok; - _ -> - tsp("finish -> stop watchdog (~p)", [Dog]), - test_server:timetrap_cancel(Dog) - end. - -%%------------------------------------------------------------------------- -%% Test cases starts here. -%%------------------------------------------------------------------------- +head() -> + [{doc, "Test http head request against local server."}]. +head(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {ok, {{_,200,_}, [_ | _], []}} = httpc:request(head, Request, [], []). +%%-------------------------------------------------------------------- +get() -> + [{doc, "Test http get request against local server"}]. +get(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} = httpc:request(get, Request, [], []), + inets_test_lib:check_body(Body), -%%------------------------------------------------------------------------- - -http_options(doc) -> - ["Test http options request against local server."]; -http_options(suite) -> - []; -http_options(Config) when is_list(Config) -> - skip("Not supported by httpd"). + {ok, {{_,200,_}, [_ | _], BinBody}} = httpc:request(get, Request, [], [{body_format, binary}]), + true = is_binary(BinBody). +%%-------------------------------------------------------------------- +post() -> + [{"Test http post request against local server. We do in this case " + "only care about the client side of the the post. The server " + "script will not actually use the post data."}]. +post(Config) when is_list(Config) -> + CGI = case test_server:os_type() of + {win32, _} -> + "/cgi-bin/cgi_echo.exe"; + _ -> + "/cgi-bin/cgi_echo" + end, -http_head(doc) -> - ["Test http head request against local server."]; -http_head(suite) -> - []; -http_head(Config) when is_list(Config) -> - tsp("http_head -> entry with" - "~n Config: ~p", [Config]), - Method = head, - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - Request = {URL, []}, - HttpOpts = [], - Opts = [], - VerifyResult = - fun({ok, {{_,200,_}, [_ | _], []}}) -> - ok; - ({ok, UnexpectedReply}) -> - tsp("http_head:verify_fun -> Unexpected Reply: " - "~n ~p", [UnexpectedReply]), - tsf({unexpected_reply, UnexpectedReply}); - ({error, Reason} = Error) -> - tsp("http_head:verify_fun -> Error reply: " - "~n Reason: ~p", [Reason]), - tsf({bad_reply, Error}) - end, - simple_request_and_verify(Config, - Method, Request, HttpOpts, Opts, VerifyResult). + URL = url(group_name(Config), CGI, Config), + %% Cgi-script expects the body length to be 100 + Body = lists:duplicate(100, "1"), -%%------------------------------------------------------------------------- + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(post, {URL, [{"expect","100-continue"}], + "text/plain", Body}, [], []), -http_get(doc) -> - ["Test http get request against local server"]; -http_get(suite) -> - []; -http_get(Config) when is_list(Config) -> - tsp("http_get -> entry with" - "~n Config: ~p", [Config]), - case ?config(local_server, Config) of - ok -> - tsp("local-server running"), - Method = get, - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - Request = {URL, []}, - Timeout = timer:seconds(1), - ConnTimeout = Timeout + timer:seconds(1), - HttpOptions1 = [{timeout, Timeout}, - {connect_timeout, ConnTimeout}], - Options1 = [], - Body = - case httpc:request(Method, Request, HttpOptions1, Options1) of - {ok, {{_,200,_}, [_ | _], ReplyBody = [_ | _]}} -> - ReplyBody; - {ok, UnexpectedReply1} -> - tsf({unexpected_reply, UnexpectedReply1}); - {error, _} = Error1 -> - tsf({bad_reply, Error1}) - end, - - %% eqvivivalent to httpc:request(get, {URL, []}, [], []), - inets_test_lib:check_body(Body), - - HttpOptions2 = [], - Options2 = [{body_format, binary}], - case httpc:request(Method, Request, HttpOptions2, Options2) of - {ok, {{_,200,_}, [_ | _], Bin}} when is_binary(Bin) -> - ok; - {ok, {{_,200,_}, [_ | _], BadBin}} -> - tsf({body_format_not_binary, BadBin}); - {ok, UnexpectedReply2} -> - tsf({unexpected_reply, UnexpectedReply2}); - {error, _} = Error2 -> - tsf({bad_reply, Error2}) - end; - _ -> - skip("Failed to start local http-server") - end. + {ok, {{_,504,_}, [_ | _], []}} = + httpc:request(post, {URL, [{"expect","100-continue"}], + "text/plain", "foobar"}, [], []). -%%------------------------------------------------------------------------- +%%-------------------------------------------------------------------- +post_stream() -> + [{"Test streaming http post request against local server. " + "We only care about the client side of the the post. " + "The server script will not actually use the post data."}]. +post_stream(Config) when is_list(Config) -> + CGI = case test_server:os_type() of + {win32, _} -> + "/cgi-bin/cgi_echo.exe"; + _ -> + "/cgi-bin/cgi_echo" + end, -http_post(doc) -> - ["Test http post request against local server. We do in this case " - "only care about the client side of the the post. The server " - "script will not actually use the post data."]; -http_post(suite) -> - []; -http_post(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - - URL = case test_server:os_type() of - {win32, _} -> - ?URL_START ++ integer_to_list(Port) ++ - "/cgi-bin/cgi_echo.exe"; - _ -> - ?URL_START ++ integer_to_list(Port) ++ - "/cgi-bin/cgi_echo" - - end, - %% Cgi-script expects the body length to be 100 - Body = lists:duplicate(100, "1"), - - {ok, {{_,200,_}, [_ | _], [_ | _]}} = - httpc:request(post, {URL, [{"expect","100-continue"}], - "text/plain", Body}, [], []), - - {ok, {{_,504,_}, [_ | _], []}} = - httpc:request(post, {URL, [{"expect","100-continue"}], - "text/plain", "foobar"}, [], []); - _ -> - skip("Failed to start local http-server") - end. + URL = url(group_name(Config), CGI, Config), -%%------------------------------------------------------------------------- -http_post_streaming(doc) -> - ["Test streaming http post request against local server. " - "We only care about the client side of the the post. " - "The server script will not actually use the post data."]; -http_post_streaming(suite) -> - []; -http_post_streaming(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - URL = case test_server:os_type() of - {win32, _} -> - ?URL_START ++ integer_to_list(Port) ++ - "/cgi-bin/cgi_echo.exe"; - _ -> - ?URL_START ++ integer_to_list(Port) ++ - "/cgi-bin/cgi_echo" - end, - %% Cgi-script expects the body length to be 100 - BodyFun = fun(0) -> - io:format("~w:http_post_streaming_fun -> " - "zero~n", [?MODULE]), - eof; - (LenLeft) -> - io:format("~w:http_post_streaming_fun -> " - "LenLeft: ~p~n", [?MODULE, LenLeft]), - {ok, lists:duplicate(10, "1"), LenLeft - 10} - end, - - {ok, {{_,200,_}, [_ | _], [_ | _]}} = - httpc:request(post, {URL, - [{"expect", "100-continue"}, - {"content-length", "100"}], - "text/plain", {BodyFun, 100}}, [], []), - - {ok, {{_,504,_}, [_ | _], []}} = - httpc:request(post, {URL, - [{"expect", "100-continue"}, - {"content-length", "10"}], - "text/plain", {BodyFun, 10}}, [], []); - - _ -> - skip("Failed to start local http-server") - end. + %% Cgi-script expects the body length to be 100 + BodyFun = fun(0) -> + eof; + (LenLeft) -> + {ok, lists:duplicate(10, "1"), LenLeft - 10} + end, + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, + {"content-length", "100"}], + "text/plain", {BodyFun, 100}}, [], []), -%%------------------------------------------------------------------------- -http_emulate_lower_versions(doc) -> - ["Perform request as 0.9 and 1.0 clients."]; -http_emulate_lower_versions(suite) -> - []; -http_emulate_lower_versions(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - {ok, Body0} = - httpc:request(get, {URL, []}, [{version, "HTTP/0.9"}], []), - inets_test_lib:check_body(Body0), - {ok, {{"HTTP/1.0", 200, _}, [_ | _], Body1 = [_ | _]}} = - httpc:request(get, {URL, []}, [{version, "HTTP/1.0"}], []), - inets_test_lib:check_body(Body1), - {ok, {{"HTTP/1.1", 200, _}, [_ | _], Body2 = [_ | _]}} = - httpc:request(get, {URL, []}, [{version, "HTTP/1.1"}], []), - inets_test_lib:check_body(Body2); - _-> - skip("Failed to start local http-server") - end. + {ok, {{_,504,_}, [_ | _], []}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, + {"content-length", "10"}], + "text/plain", {BodyFun, 10}}, [], []). +%%-------------------------------------------------------------------- +trace() -> + [{doc, "Perform a TRACE request."}]. +trace(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/trace.html", Config), []}, + case httpc:request(trace, Request, [], []) of + {ok, {{_,200,_}, [_ | _], "TRACE /trace.html" ++ _}} -> + ok; + Other -> + ct:fail({unexpected, Other}) + end. -%%------------------------------------------------------------------------- +%%-------------------------------------------------------------------- -http_relaxed(doc) -> - ["Test relaxed mode"]; -http_relaxed(suite) -> - []; -http_relaxed(Config) when is_list(Config) -> - ok = httpc:set_options([{ipv6, disabled}]), % also test the old option - %% ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), +pipeline(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {ok, _} = httpc:request(get, Request, [], [], pipeline), - URL = ?URL_START ++ integer_to_list(Port) ++ - "/missing_reason_phrase.html", - - {error, Reason} = - httpc:request(get, {URL, []}, [{relaxed, false}], []), + %% Make sure pipeline session is registerd + test_server:sleep(4000), + keep_alive_requests(Request, pipeline). - test_server:format("Not relaxed: ~p~n", [Reason]), - - {ok, {{_, 200, _}, [_ | _], [_ | _]}} = - httpc:request(get, {URL, []}, [{relaxed, true}], []), +%%-------------------------------------------------------------------- - DummyServerPid ! stop, - ok = httpc:set_options([{ipv6, enabled}]), - %% ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. +persistent_connection(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {ok, _} = httpc:request(get, Request, [], [], persistent), + %% Make sure pipeline session is registerd + test_server:sleep(4000), + keep_alive_requests(Request, persistent). %%------------------------------------------------------------------------- -http_dummy_pipe(doc) -> - ["Test pipelining code."]; -http_dummy_pipe(suite) -> - []; -http_dummy_pipe(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/foobar.html", - - test_pipeline(URL), - - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. - -http_inets_pipe(doc) -> - ["Test pipelining code."]; -http_inets_pipe(suite) -> - []; -http_inets_pipe(Config) when is_list(Config) -> - - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - test_pipeline(URL); - _ -> - skip("Failed to start local http-server") - end. +async() -> + [{doc, "Test an asynchrony http request."}]. +async(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {ok, RequestId} = + httpc:request(get, Request, [], [{sync, false}]), + Body = + receive + {http, {RequestId, {{_, 200, _}, _, BinBody}}} -> + BinBody; + {http, Msg} -> + ct:fail(Msg) + end, + inets_test_lib:check_body(binary_to_list(Body)), -test_pipeline(URL) -> - p("test_pipeline -> entry with" - "~n URL: ~p", [URL]), - - httpc:set_options([{pipeline_timeout, 50000}]), - - p("test_pipeline -> issue (async) request 1" - "~n when profile info: ~p", [httpc:info()]), - {ok, RequestIdA1} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - tsp("RequestIdA1: ~p", [RequestIdA1]), - p("test_pipeline -> RequestIdA1: ~p" - "~n when profile info: ~p", [RequestIdA1, httpc:info()]), - - %% Make sure pipeline is initiated - p("test_pipeline -> sleep some", []), - test_server:sleep(4000), - - p("test_pipeline -> issue (async) request A2, A3 and A4" - "~n when profile info: ~p", [httpc:info()]), - {ok, RequestIdA2} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - {ok, RequestIdA3} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - {ok, RequestIdA4} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - tsp("RequestIdAs => A2: ~p, A3: ~p and A4: ~p", - [RequestIdA2, RequestIdA3, RequestIdA4]), - p("test_pipeline -> RequestIds => A2: ~p, A3: ~p and A4: ~p" - "~n when profile info: ~p", - [RequestIdA2, RequestIdA3, RequestIdA4, httpc:info()]), - - p("test_pipeline -> issue (sync) request 3"), - {ok, {{_,200,_}, [_ | _], [_ | _]}} = - httpc:request(get, {URL, []}, [], []), - - p("test_pipeline -> expect reply for (async) request A1, A2, A3 and A4" - "~n when profile info: ~p", [httpc:info()]), - pipeline_await_async_reply([{RequestIdA1, a1, 200}, - {RequestIdA2, a2, 200}, - {RequestIdA3, a3, 200}, - {RequestIdA4, a4, 200}], ?MINS(1)), - - p("test_pipeline -> sleep some" - "~n when profile info: ~p", [httpc:info()]), - test_server:sleep(4000), - - p("test_pipeline -> issue (async) request B1, B2, B3 and B4" - "~n when profile info: ~p", [httpc:info()]), - {ok, RequestIdB1} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - {ok, RequestIdB2} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - {ok, RequestIdB3} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - {ok, RequestIdB4} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - tsp("RequestIdBs => B1: ~p, B2: ~p, B3: ~p and B4: ~p", - [RequestIdB1, RequestIdB2, RequestIdB3, RequestIdB4]), - p("test_pipeline -> RequestIdBs => B1: ~p, B2: ~p, B3: ~p and B4: ~p" - "~n when profile info: ~p", - [RequestIdB1, RequestIdB2, RequestIdB3, RequestIdB4, httpc:info()]), - - p("test_pipeline -> cancel (async) request B2" - "~n when profile info: ~p", [httpc:info()]), - ok = httpc:cancel_request(RequestIdB2), - - p("test_pipeline -> " - "expect *no* reply for cancelled (async) request B2 (for 3 secs)" - "~n when profile info: ~p", [httpc:info()]), + {ok, NewRequestId} = + httpc:request(get, Request, [], [{sync, false}]), + ok = httpc:cancel_request(NewRequestId), receive - {http, {RequestIdB2, _}} -> - tsf(http_cancel_request_failed) + {http, {NewRequestId, _}} -> + ct:fail(http_cancel_request_failed) after 3000 -> ok - end, - - p("test_pipeline -> expect reply for (async) request B1, B3 and B4" - "~n when profile info: ~p", [httpc:info()]), - Bodies = pipeline_await_async_reply([{RequestIdB1, b1, 200}, - {RequestIdB3, b3, 200}, - {RequestIdB4, b4, 200}], ?MINS(1)), - [{b1, Body}|_] = Bodies, - - p("test_pipeline -> check reply for (async) request B1" - "~n when profile info: ~p", [httpc:info()]), - inets_test_lib:check_body(binary_to_list(Body)), - - p("test_pipeline -> ensure no unexpected incomming" - "~n when profile info: ~p", [httpc:info()]), - receive - {http, Any} -> - tsf({unexpected_message, Any}) - after 500 -> - ok - end, - - p("test_pipeline -> done" - "~n when profile info: ~p", [httpc:info()]), - ok. - -pipeline_await_async_reply(ReqIds, Timeout) -> - pipeline_await_async_reply(ReqIds, Timeout, []). + end. +%%------------------------------------------------------------------------- +save_to_file() -> + [{doc, "Test to save the http body to a file"}]. +save_to_file(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir, Config), + FilePath = filename:join(PrivDir, "dummy.html"), + URL = url(group_name(Config), "/dummy.html", Config), + Request = {URL, []}, + {ok, saved_to_file} + = httpc:request(get, Request, [], [{stream, FilePath}]), + {ok, Bin} = file:read_file(FilePath), + {ok, {{_,200,_}, [_ | _], Body}} = httpc:request(URL), + Bin == Body. -pipeline_await_async_reply([], _, Acc) -> - lists:keysort(1, Acc); -pipeline_await_async_reply(ReqIds, Timeout, Acc) when Timeout > 0 -> - T1 = inets_test_lib:timestamp(), - p("pipeline_await_async_reply -> await replies" - "~n ReqIds: ~p" - "~n Timeout: ~p", [ReqIds, Timeout]), +%%------------------------------------------------------------------------- +save_to_file_async() -> + [{doc,"Test to save the http body to a file"}]. +save_to_file_async(Config) when is_list(Config) -> + PrivDir = ?config(priv_dir, Config), + FilePath = filename:join(PrivDir, "dummy.html"), + URL = url(group_name(Config), "/dummy.html", Config), + Request = {URL, []}, + {ok, RequestId} = httpc:request(get, Request, [], + [{stream, FilePath}, + {sync, false}]), receive - {http, {RequestId, {{_, Status, _}, _, Body}}} -> - p("pipeline_await_async_reply -> received reply for" - "~n RequestId: ~p" - "~n Status: ~p", [RequestId, Status]), - case lists:keysearch(RequestId, 1, ReqIds) of - {value, {RequestId, N, Status}} -> - p("pipeline_await_async_reply -> " - "found expected request ~w", [N]), - ReqIds2 = lists:keydelete(RequestId, 1, ReqIds), - NewTimeout = Timeout - (inets_test_lib:timestamp()-T1), - pipeline_await_async_reply(ReqIds2, NewTimeout, - [{N, Body} | Acc]); - {value, {RequestId, N, WrongStatus}} -> - p("pipeline_await_async_reply -> " - "found request ~w with wrong status", [N]), - tsf({reply_with_unexpected_status, - {RequestId, N, WrongStatus}}); - false -> - tsf({unexpected_reply, {RequestId, Status}}) - end; + {http, {RequestId, saved_to_file}} -> + ok; {http, Msg} -> - tsf({unexpected_reply, Msg}) - after Timeout -> - receive - Any -> - tsp("pipeline_await_async_reply -> " - "received unknown data after timeout: " - "~n ~p", [Any]), - tsf({timeout, {unknown, Any}}) - end - end; -pipeline_await_async_reply(ReqIds, _, Acc) -> - tsp("pipeline_await_async_reply -> " - "timeout: " - "~n ~p" - "~nwhen" - "~n ~p", [ReqIds, Acc]), - tsf({timeout, ReqIds, Acc}). - + ct:fail(Msg) + end, - + {ok, Bin} = file:read_file(FilePath), + {ok, {{_,200,_}, [_ | _], Body}} = httpc:request(URL), + Bin == Body. %%------------------------------------------------------------------------- -http_trace(doc) -> - ["Perform a TRACE request."]; -http_trace(suite) -> - []; -http_trace(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - case httpc:request(trace, {URL, []}, [], []) of - {ok, {{_,200,_}, [_ | _], "TRACE /dummy.html" ++ _}} -> - ok; - {ok, {{_,200,_}, [_ | _], WrongBody}} -> - tsf({wrong_body, WrongBody}); - {ok, WrongReply} -> - tsf({wrong_reply, WrongReply}); - Error -> - tsf({failed, Error}) - end; - _ -> - skip("Failed to start local http-server") - end. +stream() -> + [{doc, "Test the option stream for asynchrony requests"}]. +stream(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + stream_test(Request, {stream, self}). %%------------------------------------------------------------------------- -http_async(doc) -> - ["Test an asynchrony http request."]; -http_async(suite) -> - []; -http_async(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - {ok, RequestId} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - - Body = - receive - {http, {RequestId, {{_, 200, _}, _, BinBody}}} -> - BinBody; - {http, Msg} -> - tsf(Msg) - end, - - inets_test_lib:check_body(binary_to_list(Body)), - - {ok, NewRequestId} = - httpc:request(get, {URL, []}, [], [{sync, false}]), - ok = httpc:cancel_request(NewRequestId), - receive - {http, {NewRequestId, _NewResult}} -> - tsf(http_cancel_request_failed) - after 3000 -> - ok - end; - _ -> - skip("Failed to start local http-server") - end. +stream_once() -> + [{doc, "Test the option stream for asynchrony requests"}]. +stream_once(Config) when is_list(Config) -> + Request0 = {url(group_name(Config), "/dummy.html", Config), []}, + stream_test(Request0, {stream, {self, once}}), -%%------------------------------------------------------------------------- -http_save_to_file(doc) -> - ["Test to save the http body to a file"]; -http_save_to_file(suite) -> - []; -http_save_to_file(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - PrivDir = ?config(priv_dir, Config), - FilePath = filename:join(PrivDir, "dummy.html"), - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - {ok, saved_to_file} - = httpc:request(get, {URL, []}, [], [{stream, FilePath}]), - {ok, Bin} = file:read_file(FilePath), - {ok, {{_,200,_}, [_ | _], Body}} = httpc:request(URL), - Bin == Body; - _ -> - skip("Failed to start local http-server") - end. + Request1 = {url(group_name(Config), "/once.html", Config), []}, + stream_test(Request1, {stream, {self, once}}), + Request2 = {url(group_name(Config), "/once_chunked.html", Config), []}, + stream_test(Request2, {stream, {self, once}}). %%------------------------------------------------------------------------- -http_save_to_file_async(doc) -> - ["Test to save the http body to a file"]; -http_save_to_file_async(suite) -> - []; -http_save_to_file_async(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - PrivDir = ?config(priv_dir, Config), - FilePath = filename:join(PrivDir, "dummy.html"), - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - {ok, RequestId} = httpc:request(get, {URL, []}, [], - [{stream, FilePath}, - {sync, false}]), - receive - {http, {RequestId, saved_to_file}} -> - ok; - {http, Msg} -> - tsf(Msg) - end, +redirect_multiple_choises() -> + [{doc, "The user agent, selection of the most appropriate choice MAY " + "be performed automatically."}]. +redirect_multiple_choises(Config) when is_list(Config) -> + URL300 = url(group_name(Config), "/300.html", Config), - {ok, Bin} = file:read_file(FilePath), - {ok, {{_,200,_}, [_ | _], Body}} = httpc:request(URL), - Bin == Body; - _ -> - skip("Failed to start local http-server") - end. -%%------------------------------------------------------------------------- -http_headers(doc) -> - ["Use as many request headers as possible not used in proxy_headers"]; -http_headers(suite) -> - []; -http_headers(Config) when is_list(Config) -> - - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - DocRoot = ?config(doc_root, Config), - {ok, FileInfo} = - file:read_file_info(filename:join([DocRoot,"dummy.html"])), - CreatedSec = - calendar:datetime_to_gregorian_seconds( - FileInfo#file_info.mtime), - - Mod = httpd_util:rfc1123_date( - calendar:gregorian_seconds_to_datetime( - CreatedSec-1)), - - Date = httpd_util:rfc1123_date({date(), time()}), - - {ok, {{_,200,_}, [_ | _], [_ | _]}} = - httpc:request(get, {URL, [{"If-Modified-Since", - Mod}, - {"From","[email protected]"}, - {"Date", Date} - ]}, [], []), - - Mod1 = httpd_util:rfc1123_date( - calendar:gregorian_seconds_to_datetime( - CreatedSec+1)), - - {ok, {{_,200,_}, [_ | _], [_ | _]}} = - httpc:request(get, {URL, [{"If-UnModified-Since", - Mod1} - ]}, [], []), - - Tag = httpd_util:create_etag(FileInfo), - - - {ok, {{_,200,_}, [_ | _], [_ | _]}} = - httpc:request(get, {URL, [{"If-Match", - Tag} - ]}, [], []), - - {ok, {{_,200,_}, [_ | _], _}} = - httpc:request(get, {URL, [{"If-None-Match", - "NotEtag,NeihterEtag"}, - {"Connection", "Close"} - ]}, [], []), - ok; - _ -> - skip("Failed to start local http-server") - end. + catch {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, {URL300, []}, [], []), + {ok, {{_,300,_}, [_ | _], _}} = + httpc:request(get, {URL300, []}, [{autoredirect, false}], []). %%------------------------------------------------------------------------- -http_headers_dummy(doc) -> - ["Test the code for handling headers we do not want/can send " - "to a real server. Note it is not logical to send" - "all of these headers together, we only want to test that" - "the code for handling headers will not crash."]; -http_headers_dummy(suite) -> - []; -http_headers_dummy(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy_headers.html", - - Foo = http_chunk:encode("foobar") ++ - binary_to_list(http_chunk:encode_last()), - FooBar = Foo ++ "\r\n\r\nOther:inets_test\r\n\r\n", +redirect_moved_permanently() -> + [{doc, "If the 301 status code is received in response to a request other " + "than GET or HEAD, the user agent MUST NOT automatically redirect the request " + "unless it can be confirmed by the user, since this might change " + "the conditions under which the request was issued."}]. +redirect_moved_permanently(Config) when is_list(Config) -> - UserPasswd = base64:encode_to_string("Alladin:Sesame"), - Auth = "Basic " ++ UserPasswd, + URL301 = url(group_name(Config), "/301.html", Config), - %% The dummy server will ignore the headers, we only want to test - %% that the client header-handling code. This would not - %% be a vaild http-request! - {ok, {{_,200,_}, [_ | _], [_|_]}} = - httpc:request(post, - {URL, - [{"Via", - "1.0 fred, 1.1 nowhere.com (Apache/1.1)"}, - {"Warning","1#pseudonym foobar"}, - {"Vary","*"}, - {"Upgrade","HTTP/2.0"}, - {"Pragma", "1#no-cache"}, - {"Cache-Control", "no-cache"}, - {"Connection", "close"}, - {"Date", "Sat, 29 Oct 1994 19:43:31 GMT"}, - {"Accept", " text/plain; q=0.5, text/html"}, - {"Accept-Language", "en"}, - {"Accept-Encoding","chunked"}, - {"Accept-Charset", "ISO8859-1"}, - {"Authorization", Auth}, - {"Expect", "1#100-continue"}, - {"User-Agent","inets"}, - {"Transfer-Encoding","chunked"}, - {"Range", " bytes=0-499"}, - {"If-Range", "Sat, 29 Oct 1994 19:43:31 GMT"}, - {"If-Match", "*"}, - {"Content-Type", "text/plain"}, - {"Content-Encoding", "chunked"}, - {"Content-Length", "6"}, - {"Content-Language", "en"}, - {"Content-Location", "http://www.foobar.se"}, - {"Content-MD5", - "104528739076276072743283077410617235478"}, - {"Content-Range", "bytes 0-499/1234"}, - {"Allow", "GET"}, - {"Proxy-Authorization", Auth}, - {"Expires", "Sat, 29 Oct 1994 19:43:31 GMT"}, - {"Upgrade", "HTTP/2.0"}, - {"Last-Modified", "Sat, 29 Oct 1994 19:43:31 GMT"}, - {"Trailer","1#User-Agent"} - ], "text/plain", FooBar}, - [], []), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. - - -%%------------------------------------------------------------------------- -http_bad_response(doc) -> - ["Test what happens when the server does not follow the protocol"]; -http_bad_response(suite) -> - []; -http_bad_response(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/missing_crlf.html", - - URL1 = ?URL_START ++ integer_to_list(Port) ++ "/wrong_statusline.html", - - {error, timeout} = httpc:request(get, {URL, []}, [{timeout, 400}], []), - - {error, Reason} = httpc:request(URL1), - - test_server:format("Wrong Statusline: ~p~n", [Reason]), - - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, {URL301, []}, [], []), + {ok, {{_,200,_}, [_ | _], []}} + = httpc:request(head, {URL301, []}, [], []), + {ok, {{_,301,_}, [_ | _], [_|_]}} + = httpc:request(post, {URL301, [],"text/plain", "foobar"}, + [], []). %%------------------------------------------------------------------------- -ssl_head(doc) -> - ["Same as http_head/1 but over ssl sockets."]; -ssl_head(suite) -> - []; -ssl_head(Config) when is_list(Config) -> - ssl_head(ssl, Config). - -essl_head(doc) -> - ["Same as http_head/1 but over ssl sockets."]; -essl_head(suite) -> - []; -essl_head(Config) when is_list(Config) -> - ssl_head(essl, Config). - -ssl_head(SslTag, Config) -> - tsp("ssl_head -> entry with" - "~n SslTag: ~p" - "~n Config: ~p", [SslTag, Config]), - case ?config(local_ssl_server, Config) of - ok -> - DataDir = ?config(data_dir, Config), - Port = ?config(local_ssl_port, Config), - URL = ?SSL_URL_START ++ integer_to_list(Port) ++ "/dummy.html", - CertFile = filename:join(DataDir, "ssl_client_cert.pem"), - SSLOptions = [{certfile, CertFile}, {keyfile, CertFile}], - SSLConfig = - case SslTag of - ssl -> - SSLOptions; - essl -> - {essl, SSLOptions} - end, - tsp("ssl_head -> make request using: " - "~n URL: ~p" - "~n SslTag: ~p" - "~n SSLOptions: ~p", [URL, SslTag, SSLOptions]), - {ok, {{_,200, _}, [_ | _], []}} = - httpc:request(head, {URL, []}, [{ssl, SSLConfig}], []); - {ok, _} -> - skip("local http-server not started"); - _ -> - skip("SSL not started") - end. - - -%%------------------------------------------------------------------------- -ssl_get(doc) -> - ["Same as http_get/1 but over ssl sockets."]; -ssl_get(suite) -> - []; -ssl_get(Config) when is_list(Config) -> - ssl_get(ssl, Config). +redirect_found() -> + [{doc," If the 302 status code is received in response to a request other " + "than GET or HEAD, the user agent MUST NOT automatically redirect the " + "request unless it can be confirmed by the user, since this might change " + "the conditions under which the request was issued."}]. +redirect_found(Config) when is_list(Config) -> -essl_get(doc) -> - ["Same as http_get/1 but over ssl sockets."]; -essl_get(suite) -> - []; -essl_get(Config) when is_list(Config) -> - ssl_get(essl, Config). + URL302 = url(group_name(Config), "/302.html", Config), -ssl_get(SslTag, Config) when is_list(Config) -> - case ?config(local_ssl_server, Config) of - ok -> - DataDir = ?config(data_dir, Config), - Port = ?config(local_ssl_port, Config), - URL = ?SSL_URL_START ++ integer_to_list(Port) ++ "/dummy.html", - CertFile = filename:join(DataDir, "ssl_client_cert.pem"), - SSLOptions = [{certfile, CertFile}, {keyfile, CertFile}], - SSLConfig = - case SslTag of - ssl -> - SSLOptions; - essl -> - {essl, SSLOptions} - end, - tsp("ssl_get -> make request using: " - "~n URL: ~p" - "~n SslTag: ~p" - "~n SSLOptions: ~p", [URL, SslTag, SSLOptions]), - case httpc:request(get, {URL, []}, [{ssl, SSLConfig}], []) of - {ok, {{_,200, _}, [_ | _], Body = [_ | _]}} -> - inets_test_lib:check_body(Body), - ok; - {ok, {StatusLine, Headers, _Body}} -> - tsp("ssl_get -> unexpected result: " - "~n StatusLine: ~p" - "~n Headers: ~p", [StatusLine, Headers]), - tsf({unexpected_response, StatusLine, Headers}); - {error, Reason} -> - tsp("ssl_get -> request failed: " - "~n Reason: ~p", [Reason]), - tsf({request_failed, Reason}) - end; - {ok, _} -> - skip("local http-server not started"); - _ -> - skip("SSL not started") - end. + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, {URL302, []}, [], []), + {ok, {{_,200,_}, [_ | _], []}} + = httpc:request(head, {URL302, []}, [], []), + {ok, {{_,302,_}, [_ | _], [_|_]}} + = httpc:request(post, {URL302, [],"text/plain", "foobar"}, + [], []). %%------------------------------------------------------------------------- -ssl_trace(doc) -> - ["Same as http_trace/1 but over ssl sockets."]; -ssl_trace(suite) -> - []; -ssl_trace(Config) when is_list(Config) -> - ssl_trace(ssl, Config). +redirect_see_other() -> + [{doc, "The different URI SHOULD be given by the Location field in the response. " + "Unless the request method was HEAD, the entity of the response SHOULD contain a short " + "hypertext note with a hyperlink to the new URI(s). "}]. +redirect_see_other(Config) when is_list(Config) -> -essl_trace(doc) -> - ["Same as http_trace/1 but over ssl sockets."]; -essl_trace(suite) -> - []; -essl_trace(Config) when is_list(Config) -> - ssl_trace(essl, Config). + URL303 = url(group_name(Config), "/303.html", Config), -ssl_trace(SslTag, Config) when is_list(Config) -> - case ?config(local_ssl_server, Config) of - ok -> - DataDir = ?config(data_dir, Config), - Port = ?config(local_ssl_port, Config), - URL = ?SSL_URL_START ++ integer_to_list(Port) ++ "/dummy.html", - CertFile = filename:join(DataDir, "ssl_client_cert.pem"), - SSLOptions = [{certfile, CertFile}, {keyfile, CertFile}], - SSLConfig = - case SslTag of - ssl -> - SSLOptions; - essl -> - {essl, SSLOptions} - end, - tsp("ssl_trace -> make request using: " - "~n URL: ~p" - "~n SslTag: ~p" - "~n SSLOptions: ~p", [URL, SslTag, SSLOptions]), - case httpc:request(trace, {URL, []}, [{ssl, SSLConfig}], []) of - {ok, {{_,200, _}, [_ | _], "TRACE /dummy.html" ++ _}} -> - ok; - {ok, {{_,200,_}, [_ | _], WrongBody}} -> - tsf({wrong_body, WrongBody}); - {ok, WrongReply} -> - tsf({wrong_reply, WrongReply}); - Error -> - tsf({failed, Error}) - end; - {ok, _} -> - skip("local http-server not started"); - _ -> - skip("SSL not started") - end. + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, {URL303, []}, [], []), + {ok, {{_,200,_}, [_ | _], []}} + = httpc:request(head, {URL303, []}, [], []), + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(post, {URL303, [],"text/plain", "foobar"}, + [], []). %%------------------------------------------------------------------------- -http_redirect(doc) -> - ["Test redirect with dummy server as httpd does not implement" - " server redirect"]; -http_redirect(suite) -> - []; -http_redirect(Config) when is_list(Config) -> - tsp("http_redirect -> entry with" - "~n Config: ~p", [Config]), - case ?config(local_server, Config) of - ok -> - %% tsp("http_redirect -> set ipfamily option to inet"), - %% ok = httpc:set_options([{ipfamily, inet}]), +redirect_temporary_redirect() -> + [{doc," If the 307 status code is received in response to a request other " + "than GET or HEAD, the user agent MUST NOT automatically redirect the request " + "unless it can be confirmed by the user, since this might change " + "the conditions under which the request was issued."}]. +redirect_temporary_redirect(Config) when is_list(Config) -> - tsp("http_redirect -> start dummy server inet"), - {DummyServerPid, Port} = dummy_server(ipv4), - tsp("http_redirect -> server port = ~p", [Port]), - - URL300 = ?URL_START ++ integer_to_list(Port) ++ "/300.html", - - tsp("http_redirect -> issue request 1: " - "~n ~p", [URL300]), - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL300, []}, [], []), - - tsp("http_redirect -> issue request 2: " - "~n ~p", [URL300]), - {ok, {{_,300,_}, [_ | _], _}} = - httpc:request(get, {URL300, []}, [{autoredirect, false}], []), - - URL301 = ?URL_START ++ integer_to_list(Port) ++ "/301.html", - - tsp("http_redirect -> issue request 3: " - "~n ~p", [URL301]), - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL301, []}, [], []), - - tsp("http_redirect -> issue request 4: " - "~n ~p", [URL301]), - {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL301, []}, [], []), - - tsp("http_redirect -> issue request 5: " - "~n ~p", [URL301]), - {ok, {{_,301,_}, [_ | _], [_|_]}} - = httpc:request(post, {URL301, [],"text/plain", "foobar"}, - [], []), - - URL302 = ?URL_START ++ integer_to_list(Port) ++ "/302.html", - - tsp("http_redirect -> issue request 6: " - "~n ~p", [URL302]), - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL302, []}, [], []), - case httpc:request(get, {URL302, []}, [], []) of - {ok, Reply7} -> - case Reply7 of - {{_,200,_}, [_ | _], [_|_]} -> - tsp("http_redirect -> " - "expected reply for request 7"), - ok; - {StatusLine, Headers, Body} -> - tsp("http_redirect -> " - "unexpected reply for request 7: " - "~n StatusLine: ~p" - "~n Headers: ~p" - "~n Body: ~p", - [StatusLine, Headers, Body]), - tsf({unexpected_reply, Reply7}) - end; - Error7 -> - tsp("http_redirect -> " - "unexpected result for request 7: " - "~n Error7: ~p", - [Error7]), - tsf({unexpected_result, Error7}) - end, - - tsp("http_redirect -> issue request 7: " - "~n ~p", [URL302]), - {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL302, []}, [], []), - - tsp("http_redirect -> issue request 8: " - "~n ~p", [URL302]), - {ok, {{_,302,_}, [_ | _], [_|_]}} - = httpc:request(post, {URL302, [],"text/plain", "foobar"}, - [], []), - - URL303 = ?URL_START ++ integer_to_list(Port) ++ "/303.html", - - tsp("http_redirect -> issue request 9: " - "~n ~p", [URL303]), - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL303, []}, [], []), - - tsp("http_redirect -> issue request 10: " - "~n ~p", [URL303]), - {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL303, []}, [], []), - - tsp("http_redirect -> issue request 11: " - "~n ~p", [URL303]), - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(post, {URL303, [],"text/plain", "foobar"}, - [], []), - - URL307 = ?URL_START ++ integer_to_list(Port) ++ "/307.html", - - tsp("http_redirect -> issue request 12: " - "~n ~p", [URL307]), - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URL307, []}, [], []), - - tsp("http_redirect -> issue request 13: " - "~n ~p", [URL307]), - {ok, {{_,200,_}, [_ | _], []}} - = httpc:request(head, {URL307, []}, [], []), - - tsp("http_redirect -> issue request 14: " - "~n ~p", [URL307]), - {ok, {{_,307,_}, [_ | _], [_|_]}} - = httpc:request(post, {URL307, [],"text/plain", "foobar"}, - [], []), - - tsp("http_redirect -> stop dummy server"), - DummyServerPid ! stop, - tsp("http_redirect -> reset ipfamily option (to inet6fb4)"), - ok = httpc:set_options([{ipfamily, inet6fb4}]), - tsp("http_redirect -> done"), - ok; - - _ -> - skip("Failed to start local http-server") - end. + URL307 = url(group_name(Config), "/307.html", Config), + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, {URL307, []}, [], []), + {ok, {{_,200,_}, [_ | _], []}} + = httpc:request(head, {URL307, []}, [], []), -%%------------------------------------------------------------------------- -http_redirect_loop(doc) -> - ["Test redirect loop detection"]; -http_redirect_loop(suite) -> - []; -http_redirect_loop(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/redirectloop.html", - - {ok, {{_,300,_}, [_ | _], _}} - = httpc:request(get, {URL, []}, [], []), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + {ok, {{_,307,_}, [_ | _], [_|_]}} + = httpc:request(post, {URL307, [],"text/plain", "foobar"}, + [], []). %%------------------------------------------------------------------------- -http_internal_server_error(doc) -> - ["Test 50X codes"]; -http_internal_server_error(suite) -> - []; -http_internal_server_error(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL500 = ?URL_START ++ integer_to_list(Port) ++ "/500.html", - - {ok, {{_,500,_}, [_ | _], _}} - = httpc:request(get, {URL500, []}, [], []), +redirect_loop() -> + [{"doc, Test redirect loop detection"}]. +redirect_loop(Config) when is_list(Config) -> + URL = url(group_name(Config), "/redirectloop.html", Config), - URL503 = ?URL_START ++ integer_to_list(Port) ++ "/503.html", + {ok, {{_,300,_}, [_ | _], _}} + = httpc:request(get, {URL, []}, [], []). - %% Used to be able to make the service available after retry. - ets:new(unavailable, [named_table, public, set]), - ets:insert(unavailable, {503, unavailable}), - - {ok, {{_,200, _}, [_ | _], [_|_]}} = - httpc:request(get, {URL503, []}, [], []), - - ets:insert(unavailable, {503, long_unavailable}), +%%------------------------------------------------------------------------- +cookie() -> + [{doc, "Test cookies."}]. +cookie(Config) when is_list(Config) -> + ok = httpc:set_options([{cookies, enabled}]), - {ok, {{_,503, _}, [_ | _], [_|_]}} = - httpc:request(get, {URL503, []}, [], []), + Request0 = {url(group_name(Config), "/cookie.html", Config), []}, - ets:delete(unavailable), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, Request0, [], []), + %% Populate table to be used by the "dummy" server + ets:new(cookie, [named_table, public, set]), + ets:insert(cookie, {cookies, true}), -%%------------------------------------------------------------------------- -http_userinfo(doc) -> - ["Test user info e.i. http://user:passwd@host:port/"]; -http_userinfo(suite) -> - []; -http_userinfo(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), + Request1 = {url(group_name(Config), "/", Config), []}, - {DummyServerPid, Port} = dummy_server(ipv4), - - URLAuth = "http://alladin:sesame@localhost:" - ++ integer_to_list(Port) ++ "/userinfo.html", - - {ok, {{_,200,_}, [_ | _], _}} - = httpc:request(get, {URLAuth, []}, [], []), + {ok, {{_,200,_}, [_ | _], [_|_]}} + = httpc:request(get, Request1, [], []), - URLUnAuth = "http://alladin:foobar@localhost:" - ++ integer_to_list(Port) ++ "/userinfo.html", - - {ok, {{_,401, _}, [_ | _], _}} = - httpc:request(get, {URLUnAuth, []}, [], []), - - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + ets:delete(cookie), + ok = httpc:set_options([{cookies, disabled}]). +%%------------------------------------------------------------------------- +headers_as_is(doc) -> + ["Test the option headers_as_is"]; +headers_as_is(Config) when is_list(Config) -> + URL = url(group_name(Config), "/dummy.html", Config), + {ok, {{_,200,_}, [_|_], [_|_]}} = + httpc:request(get, {URL, [{"Host", "localhost"},{"Te", ""}]}, + [], [{headers_as_is, true}]), + {ok, {{_,400,_}, [_|_], [_|_]}} = + httpc:request(get, {URL, [{"Te", ""}]},[], [{headers_as_is, true}]). %%------------------------------------------------------------------------- -http_cookie(doc) -> - ["Test cookies."]; -http_cookie(suite) -> - []; -http_cookie(Config) when is_list(Config) -> - ok = httpc:set_options([{cookies, enabled}, {ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), + +userinfo(doc) -> + [{doc, "Test user info e.i. http://user:passwd@host:port/"}]; +userinfo(Config) when is_list(Config) -> - URLStart = ?URL_START - ++ integer_to_list(Port), + {ok,Host} = inet:gethostname(), - URLCookie = URLStart ++ "/cookie.html", - - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URLCookie, []}, [], []), + URLAuth = url(group_name(Config), "alladin:sesame@" ++ Host ++ ":","/userinfo.html", Config), - ets:new(cookie, [named_table, public, set]), - ets:insert(cookie, {cookies, true}), + {ok, {{_,200,_}, [_ | _], _}} + = httpc:request(get, {URLAuth, []}, [], []), - {ok, {{_,200,_}, [_ | _], [_|_]}} - = httpc:request(get, {URLStart ++ "/", []}, [], []), - - ets:delete(cookie), + URLUnAuth = url(group_name(Config), "alladin:foobar@" ++ Host ++ ":","/userinfo.html", Config), - ok = httpc:set_options([{cookies, disabled}]), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + {ok, {{_,401, _}, [_ | _], _}} = + httpc:request(get, {URLUnAuth, []}, [], []). %%------------------------------------------------------------------------- -http_server_does_not_exist(doc) -> - ["Test that we get an error message back when the server " - "does note exist."]; -http_server_does_not_exist(suite) -> - []; -http_server_does_not_exist(Config) when is_list(Config) -> - {error, _} = - httpc:request(get, {"http://localhost:" ++ - integer_to_list(?NOT_IN_USE_PORT) - ++ "/", []},[], []), - ok. - -%%------------------------------------------------------------------------- page_does_not_exist(doc) -> ["Test that we get a 404 when the page is not found."]; -page_does_not_exist(suite) -> - []; page_does_not_exist(Config) when is_list(Config) -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/doesnotexist.html", - {ok, {{_,404,_}, [_ | _], [_ | _]}} - = httpc:request(get, {URL, []}, [], []), - ok. - - + URL = url(group_name(Config), "/doesnotexist.html", Config), + {ok, {{_,404,_}, [_ | _], [_ | _]}} + = httpc:request(get, {URL, []}, [], []). %%------------------------------------------------------------------------- -http_stream(doc) -> - ["Test the option stream for asynchrony requests"]; -http_stream(suite) -> - []; -http_stream(Config) when is_list(Config) -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - {ok, {{_,200,_}, [_ | _], Body}} = - httpc:request(get, {URL, []}, [], []), - - {ok, RequestId} = - httpc:request(get, {URL, []}, [], [{sync, false}, - {stream, self}]), - - receive - {http, {RequestId, stream_start, _Headers}} -> - ok; - {http, Msg} -> - tsf(Msg) - end, - - StreamedBody = receive_streamed_body(RequestId, <<>>), - - Body == binary_to_list(StreamedBody). - +streaming_error(doc) -> + [{doc, "Only async requests can be stremed - Solves OTP-8056"}]; +streaming_error(Config) when is_list(Config) -> + Method = get, + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {error, streaming_error} = httpc:request(Method, Request, + [], [{sync, true}, {stream, {self, once}}]), + {error, streaming_error} = httpc:request(Method, Request, + [], [{sync, true}, {stream, self}]). %%------------------------------------------------------------------------- -http_stream_once(doc) -> - ["Test the option stream for asynchrony requests"]; -http_stream_once(suite) -> - []; -http_stream_once(Config) when is_list(Config) -> - p("http_stream_once -> entry with" - "~n Config: ~p", [Config]), - - p("http_stream_once -> set ipfamily to inet", []), - ok = httpc:set_options([{ipfamily, inet}]), - p("http_stream_once -> start dummy server", []), - {DummyServerPid, Port} = dummy_server(ipv4), - - PortStr = integer_to_list(Port), - p("http_stream_once -> once", []), - once(?URL_START ++ PortStr ++ "/once.html"), - p("http_stream_once -> once_chunked", []), - once(?URL_START ++ PortStr ++ "/once_chunked.html"), - p("http_stream_once -> dummy", []), - once(?URL_START ++ PortStr ++ "/dummy.html"), - - p("http_stream_once -> stop dummy server", []), - DummyServerPid ! stop, - p("http_stream_once -> set ipfamily to inet6fb4", []), - ok = httpc:set_options([{ipfamily, inet6fb4}]), - p("http_stream_once -> done", []), - ok. - -once(URL) -> - p("once -> issue sync request for ~p", [URL]), - {ok, {{_,200,_}, [_ | _], Body}} = - httpc:request(get, {URL, []}, [], []), - - p("once -> issue async (self stream) request for ~p", [URL]), - {ok, RequestId} = - httpc:request(get, {URL, []}, [], [{sync, false}, - {stream, {self, once}}]), - - p("once -> await stream_start reply for (async) request ~p", [RequestId]), - NewPid = - receive - {http, {RequestId, stream_start, _Headers, Pid}} -> - p("once -> received stream_start reply for (async) request ~p: ~p", - [RequestId, Pid]), - Pid; - {http, Msg} -> - tsf(Msg) - end, - - tsp("once -> request handler: ~p", [NewPid]), - - p("once -> await stream reply for (async) request ~p", [RequestId]), - BodyPart = - receive - {http, {RequestId, stream, BinBodyPart}} -> - p("once -> received stream reply for (async) request ~p: " - "~n~p", [RequestId, binary_to_list(BinBodyPart)]), - BinBodyPart - end, - - tsp("once -> first body part '~p' received", [binary_to_list(BodyPart)]), - - StreamedBody = receive_streamed_body(RequestId, BinBodyPart, NewPid), - - Body = binary_to_list(StreamedBody), - - p("once -> done when Bode: ~p", [Body]), - ok. - - +server_does_not_exist(doc) -> + [{doc, "Test that we get an error message back when the server " + "does note exist."}]; +server_does_not_exist(Config) when is_list(Config) -> + {error, _} = + httpc:request(get, {"http://localhost:" ++ + integer_to_list(?NOT_IN_USE_PORT) + ++ "/", []},[], []). %%------------------------------------------------------------------------- -parse_url(doc) -> - ["Test that an url is parsed correctly"]; -parse_url(suite) -> - []; -parse_url(Config) when is_list(Config) -> - %% ipv6 - {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = - http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html"), - {ok, {http,[],"[2010:836B:4179::836B:4179]",80,"/foobar.html",[]}} = - http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", - [{ipv6_host_with_brackets, true}]), - {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = - http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", - [{ipv6_host_with_brackets, false}]), - {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = - http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", - [{foo, false}]), - {error, - {malformed_url, _, "http://2010:836B:4179::836B:4179/foobar.html"}} = - http_uri:parse("http://2010:836B:4179::836B:4179/foobar.html"), - - %% ipv4 - {ok, {http,[],"127.0.0.1",80,"/foobar.html",[]}} = - http_uri:parse("http://127.0.0.1/foobar.html"), - - %% host - {ok, {http,[],"localhost",8888,"/foobar.html",[]}} = - http_uri:parse("http://localhost:8888/foobar.html"), - - %% Userinfo - {ok, {http,"nisse:foobar","localhost",8888,"/foobar.html",[]}} = - http_uri:parse("http://nisse:foobar@localhost:8888/foobar.html"), - - %% Scheme error - {error, no_scheme} = http_uri:parse("localhost/foobar.html"), - {error, {malformed_url, _, _}} = - http_uri:parse("localhost:8888/foobar.html"), - - %% Query - {ok, {http,[],"localhost",8888,"/foobar.html","?foo=bar&foobar=42"}} = - http_uri:parse("http://localhost:8888/foobar.html?foo=bar&foobar=42"), - - %% Esc chars - {ok, {http,[],"www.somedomain.com",80,"/%2Eabc",[]}} = - http_uri:parse("http://www.somedomain.com/%2Eabc"), - {ok, {http,[],"www.somedomain.com",80,"/%252Eabc",[]}} = - http_uri:parse("http://www.somedomain.com/%252Eabc"), - {ok, {http,[],"www.somedomain.com",80,"/%25abc",[]}} = - http_uri:parse("http://www.somedomain.com/%25abc"), - {ok, {http,[],"www.somedomain.com",80,"/%25abc", "?foo=bar"}} = - http_uri:parse("http://www.somedomain.com/%25abc?foo=bar"), - - - ok. +no_content_204(doc) -> + ["Test the case that the HTTP 204 no content header - Solves OTP 6982"]; +no_content_204(Config) when is_list(Config) -> + URL = url(group_name(Config), "/no_content.html", Config), + {ok, {{_,204,_}, [], []}} = httpc:request(URL). %%------------------------------------------------------------------------- -ipv6_ipcomm() -> - [{require, ipv6_hosts}]. -ipv6_ipcomm(doc) -> - ["Test ip_comm ipv6."]; -ipv6_ipcomm(suite) -> - []; -ipv6_ipcomm(Config) when is_list(Config) -> - HTTPOptions = [], - SocketType = ip_comm, - Scheme = "http", - Extra = [], - ipv6(SocketType, Scheme, HTTPOptions, Extra, Config). - - +tolerate_missing_CR() -> + [{doc, "Test the case that the HTTP server uses only LF instead of CRLF" + "as delimitor. Solves OTP-7304"}]. +tolerate_missing_CR(Config) when is_list(Config) -> + URL = url(group_name(Config), "/missing_CR.html", Config), + {ok, {{_,200,_}, _, [_ | _]}} = httpc:request(URL). %%------------------------------------------------------------------------- -ipv6_essl() -> - [{require, ipv6_hosts}]. -ipv6_essl(doc) -> - ["Test essl ipv6."]; -ipv6_essl(suite) -> - []; -ipv6_essl(Config) when is_list(Config) -> - DataDir = ?config(data_dir, Config), - CertFile = filename:join(DataDir, "ssl_client_cert.pem"), - SSLOptions = [{certfile, CertFile}, {keyfile, CertFile}], - SSLConfig = {essl, SSLOptions}, - tsp("ssl_ipv6 -> make request using: " - "~n SSLOptions: ~p", [SSLOptions]), - HTTPOptions = [{ssl, SSLConfig}], - SocketType = essl, - Scheme = "https", - Extra = SSLOptions, - ipv6(SocketType, Scheme, HTTPOptions, Extra, Config). - +empty_body() -> + [{doc, "An empty body was not returned directly. There was a delay for several" + "seconds. Solves OTP-6243."}]. +empty_body(Config) when is_list(Config) -> + URL = url(group_name(Config), "/empty.html", Config), + {ok, {{_,200,_}, [_ | _], []}} = + httpc:request(get, {URL, []}, [{timeout, 500}], []). %%------------------------------------------------------------------------- -ipv6(SocketType, Scheme, HTTPOptions, Extra, Config) -> - %% Check if we are a IPv6 host - tsp("ipv6 -> verify ipv6 support"), - case inets_test_lib:has_ipv6_support(Config) of - {ok, Addr} -> - tsp("ipv6 -> ipv6 supported: ~p", [Addr]), - {DummyServerPid, Port} = dummy_server(SocketType, ipv6, Extra), - Profile = ?config(profile, Config), - URL = - Scheme ++ - "://[" ++ http_transport:ipv6_name(Addr) ++ "]:" ++ - integer_to_list(Port) ++ "/foobar.html", - tsp("ipv6 -> issue request with: " - "~n URL: ~p" - "~n HTTPOptions: ~p", [URL, HTTPOptions]), - case httpc:request(get, {URL, []}, HTTPOptions, [], Profile) of - {ok, {{_,200,_}, [_ | _], [_|_]}} -> - tsp("ipv6 -> expected result"), - DummyServerPid ! stop, - ok; - {ok, Unexpected} -> - tsp("ipv6 -> unexpected result: " - "~n ~p", [Unexpected]), - DummyServerPid ! stop, - tsf({unexpected_result, Unexpected}); - {error, Reason} -> - tsp("ipv6 -> error: " - "~n Reason: ~p", [Reason]), - DummyServerPid ! stop, - tsf(Reason) - end, - ok; - _ -> - tsp("ipv6 -> ipv6 not supported"), - skip("Host does not support IPv6") - end. - +transfer_encoding() -> + [{doc, "Transfer encoding is case insensitive. Solves OTP-6807"}]. +transfer_encoding(Config) when is_list(Config) -> + URL = url(group_name(Config), "/capital_transfer_encoding.html", Config), + {ok, {{_,200,_}, [_|_], [_ | _]}} = httpc:request(URL). %%------------------------------------------------------------------------- -headers_as_is(doc) -> - ["Test the option headers_as_is"]; -headers_as_is(suite) -> - []; -headers_as_is(Config) when is_list(Config) -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - {ok, {{_,200,_}, [_|_], [_|_]}} = - httpc:request(get, {URL, [{"Host", "localhost"},{"Te", ""}]}, - [], [{headers_as_is, true}]), - - {ok, {{_,400,_}, [_|_], [_|_]}} = - httpc:request(get, {URL, [{"Te", ""}]},[], [{headers_as_is, true}]), - ok. - +empty_response_header() -> + [{doc, "Test the case that the HTTP server does not send any headers. Solves OTP-6830"}]. +empty_response_header(Config) when is_list(Config) -> + URL = url(group_name(Config), "/no_headers.html", Config), + {ok, {{_,200,_}, [], [_ | _]}} = httpc:request(URL). %%------------------------------------------------------------------------- -selecting_session(doc) -> - ["Test selection of sessions - OTP-9847"]; -selecting_session(suite) -> - []; -selecting_session(Config) when is_list(Config) -> - tsp("selecting_session -> entry with" - "~n Config: ~p", [Config]), - - tsp("selecting_session -> set ipfamily to inet"), - ok = httpc:set_options([{ipfamily, inet}]), - - tsp("selecting_session -> start server"), - {ServerPid, Port} = otp_9847_server(), - - PortStr = integer_to_list(Port), - URL = ?URL_START ++ PortStr ++ "/index.html", - - tsp("selecting_session -> issue the first batch (three) requests"), - lists:foreach(fun(P) -> - tsp("selecting_session:fun1 -> " - "send stop request to ~p", [P]), - P ! stop - end, - reqs(URL, ServerPid, 3, 3, false)), - tsp("selecting_session -> sleep some (1) to make sure nothing lingers"), - ?SLEEP(5000), - tsp("selecting_session -> " - "instruct the server to reply to the first request"), - ServerPid ! {answer, true}, - receive - {answer, true} -> - tsp("selecting_session -> " - "received ack from server to reply to the first request"), - ok - end, - tsp("selecting_session -> issue the second batch (four) requests"), - lists:foreach(fun(P) -> - tsp("selecting_session:fun2 -> " - "send stop request to ~p", [P]), - P ! stop - end, - reqs(URL, ServerPid, 4, 1, true)), - tsp("selecting_session -> sleep some (2) to make sure nothing lingers"), - ?SLEEP(5000), - - tsp("selecting_session -> stop server"), - ServerPid ! stop, - tsp("selecting_session -> set ipfamily (back) to inet6fb4"), - ok = httpc:set_options([{ipfamily, inet6fb4}]), - tsp("selecting_session -> done"), - ok. - -reqs(URL, ServerPid, NumReqs, NumHandlers, InitialSync) -> - tsp("reqs -> entry with" - "~n URL: ~p" - "~n ServerPid: ~w" - "~n NumReqs: ~w" - "~n NumHandlers: ~w" - "~n InitialSync: ~w", - [URL, ServerPid, NumReqs, NumHandlers, InitialSync]), - Handlers = reqs2(URL, NumReqs, [], InitialSync), - tsp("reqs -> " - "~n Handlers: ~w", [Handlers]), - case length(Handlers) of - NumHandlers -> - tsp("reqs -> " - "~n NumHandlers: ~w", [NumHandlers]), - ServerPid ! num_handlers, - receive - {num_handlers, NumHandlers} -> - tsp("reqs -> received num_handlers with" - "~n NumHandlers: ~w", [NumHandlers]), - Handlers; - {num_handlers, WrongNumHandlers} -> - tsp("reqs -> received num_handlers with" - "~n WrongNumHandlers: ~w", [WrongNumHandlers]), - exit({wrong_num_handlers1, WrongNumHandlers, NumHandlers}) - end; - WrongNumHandlers -> - tsp("reqs -> " - "~n WrongNumHandlers: ~w", [WrongNumHandlers]), - exit({wrong_num_handlers2, WrongNumHandlers, NumHandlers}) - end. - - -reqs2(_URL, 0, Acc, _Sync) -> - lists:reverse(Acc); -reqs2(URL, Num, Acc, Sync) -> - tsp("reqs2 -> entry with" - "~n Num: ~w" - "~n Sync: ~w", [Num, Sync]), - case httpc:request(get, {URL, []}, [], [{sync, Sync}]) of - {ok, _Reply} -> - tsp("reqs2 -> successful request: ~p", [_Reply]), - receive - {handler, Handler, _Manager} -> - %% This is when a new handler is created - tsp("reqs2 -> received handler: ~p", [Handler]), - case lists:member(Handler, Acc) of - true -> - tsp("reqs2 -> duplicate handler"), - exit({duplicate_handler, Handler, Num, Acc}); - false -> - tsp("reqs2 -> wait for data ack"), - receive - {data_received, Handler} -> - tsp("reqs2 -> " - "received data ack from ~p", [Handler]), - case Sync of - true -> - reqs2(URL, Num-1, [Handler|Acc], - false); - false -> - reqs2(URL, Num-1, [Handler|Acc], - Sync) - end - end - end; - - {data_received, Handler} -> - tsp("reqs2 -> " - "received data ack from ~p", [Handler]), - reqs2(URL, Num-1, Acc, false) +bad_response(doc) -> + [{doc, "Test what happens when the server does not follow the protocol"}]; - end; +bad_response(Config) when is_list(Config) -> - {error, Reason} -> - tsp("reqs2 -> request ~w failed: ~p", [Num, Reason]), - exit({request_failed, Reason, Num, Acc}) - end. + URL0 = url(group_name(Config), "/missing_crlf.html", Config), + URL1 = url(group_name(Config), "/wrong_statusline.html", Config), -otp_9847_server() -> - TC = self(), - Pid = spawn_link(fun() -> otp_9847_server_init(TC) end), - receive - {port, Port} -> - {Pid, Port} - end. + {error, timeout} = httpc:request(get, {URL0, []}, [{timeout, 400}], []), + {error, Reason} = httpc:request(URL1), -otp_9847_server_init(TC) -> - tsp("otp_9847_server_init -> entry with" - "~n TC: ~p", [TC]), - {ok, ListenSocket} = - gen_tcp:listen(0, [binary, inet, {packet, 0}, - {reuseaddr,true}, - {active, false}]), - tsp("otp_9847_server_init -> listen socket created: " - "~n ListenSocket: ~p", [ListenSocket]), - {ok, Port} = inet:port(ListenSocket), - tsp("otp_9847_server_init -> Port: ~p", [Port]), - TC ! {port, Port}, - otp_9847_server_main(TC, ListenSocket, false, []). - -otp_9847_server_main(TC, ListenSocket, Answer, Handlers) -> - tsp("otp_9847_server_main -> entry with" - "~n TC: ~p" - "~n ListenSocket: ~p" - "~n Answer: ~p" - "~n Handlers: ~p", [TC, ListenSocket, Answer, Handlers]), - case gen_tcp:accept(ListenSocket, 1000) of - {ok, Sock} -> - tsp("otp_9847_server_main -> accepted" - "~n Sock: ~p", [Sock]), - {Handler, Mon, Port} = otp_9847_handler(TC, Sock, Answer), - tsp("otp_9847_server_main -> handler ~p created for ~w", - [Handler, Port]), - gen_tcp:controlling_process(Sock, Handler), - tsp("otp_9847_server_main -> control transfer"), - Handler ! owner, - tsp("otp_9847_server_main -> " - "handler ~p informed of owner transfer", [Handler]), - TC ! {handler, Handler, self()}, - tsp("otp_9847_server_main -> " - "TC ~p informed of handler ~p", [TC, Handler]), - otp_9847_server_main(TC, ListenSocket, Answer, - [{Handler, Mon, Sock, Port}|Handlers]); + ct:print("Wrong Statusline: ~p~n", [Reason]). +%%------------------------------------------------------------------------- - {error, timeout} -> - tsp("otp_9847_server_main -> timeout"), - receive - {answer, true} -> - tsp("otp_9847_server_main -> received answer request"), - TC ! {answer, true}, - otp_9847_server_main(TC, ListenSocket, true, Handlers); - - {'DOWN', _Mon, process, Pid, _Reason} -> - %% Could be one of the handlers - tsp("otp_9847_server_main -> received DOWN for ~p", [Pid]), - otp_9847_server_main(TC, ListenSocket, Answer, - lists:keydelete(Pid, 1, Handlers)); - - num_handlers -> - tsp("otp_9847_server_main -> " - "received request for number of handlers (~w)", - [length(Handlers)]), - TC ! {num_handlers, length(Handlers)}, - otp_9847_server_main(TC, ListenSocket, Answer, Handlers); +internal_server_error(doc) -> + ["Test 50X codes"]; +internal_server_error(Config) when is_list(Config) -> - stop -> - tsp("otp_9847_server_main -> received stop request"), - %% Stop all handlers (just in case) - Pids = [Handler || {Handler, _, _} <- Handlers], - lists:foreach(fun(Pid) -> Pid ! stop end, Pids), - exit(normal); + URL500 = url(group_name(Config), "/500.html", Config), - Any -> - tsp("otp_9847_server_main -> received" - "~n Any: ~p", [Any]), - exit({crap, Any}) + {ok, {{_,500,_}, [_ | _], _}} + = httpc:request(get, {URL500, []}, [], []), - after 0 -> - tsp("otp_9847_server_main -> nothing in queue"), - otp_9847_server_main(TC, ListenSocket, Answer, Handlers) - end; + URL503 = url(group_name(Config), "/503.html", Config), - Error -> - exit(Error) - end. + %% Used to be able to make the service available after retry. + ets:new(unavailable, [named_table, public, set]), + ets:insert(unavailable, {503, unavailable}), + {ok, {{_,200, _}, [_ | _], [_|_]}} = + httpc:request(get, {URL503, []}, [], []), -otp_9847_handler(TC, Sock, Answer) -> - tsp("otp_9847_handler -> entry with" - "~n TC: ~p" - "~n Sock: ~p" - "~n Answer: ~p", [TC, Sock, Answer]), - Self = self(), - {Pid, Mon} = - spawn_opt(fun() -> - otp_9847_handler_init(TC, Self, Sock, Answer) - end, - [monitor]), - receive - {port, Port} -> - tsp("otp_9847_handler -> received port message (from ~p)" - "~n Port: ~p", [Pid, Port]), - {Pid, Mon, Port} - end. - + ets:insert(unavailable, {503, long_unavailable}), -otp_9847_handler_init(TC, Server, Sock, Answer) -> - tsp("otp_9847_handler_init -> entry with" - "~n TC: ~p" - "~n Server: ~p" - "~n Sock: ~p" - "~n Answer: ~p", [TC, Server, Sock, Answer]), - {ok, Port} = inet:port(Sock), - Server ! {port, Port}, - receive - owner -> - tsp("otp_9847_handler_init -> " - "received owner message - activate socket"), - inet:setopts(Sock, [{active, true}]), - otp_9847_handler_main(TC, Server, Sock, Answer, [?HTTP_MAX_HEADER_SIZE]) - end. - -otp_9847_handler_main(TC, Server, Sock, Answer, ParseArgs) -> - tsp("otp_9847_handler_main -> entry with" - "~n TC: ~p" - "~n Server: ~p" - "~n Sock: ~p" - "~n Answer: ~p" - "~n ParseArgs: ~p", [TC, Server, Sock, Answer, ParseArgs]), - receive - stop -> - tsp("otp_9847_handler_main -> received stop request"), - exit(normal); - - {tcp, Sock, _Data} when Answer =:= false -> - tsp("otp_9847_handler_main -> received tcp data - no answer"), - TC ! {data_received, self()}, - inet:setopts(Sock, [{active, true}]), - %% Ignore all data - otp_9847_handler_main(TC, Server, Sock, Answer, ParseArgs); - - {tcp, Sock, Data} when Answer =:= true -> - tsp("otp_9847_handler_main -> received tcp data - answer"), - TC ! {data_received, self()}, - inet:setopts(Sock, [{active, true}]), - NewParseArgs = otp_9847_handler_request(Sock, [Data|ParseArgs]), - otp_9847_handler_main(TC, Server, Sock, Answer, NewParseArgs); - - {tcp_closed, Sock} -> - tsp("otp_9847_handler_main -> received tcp socket closed"), - exit(normal); - - {tcp_error, Sock, Reason} -> - tsp("otp_9847_handler_main -> socket error: ~p", [Reason]), - (catch gen_tcp:close(Sock)), - exit(normal) - - %% after 30000 -> - %% gen_tcp:close(Sock), - %% exit(normal) - end. - -otp_9847_handler_request(Sock, Args) -> - Msg = - case httpd_request:parse(Args) of - {ok, {_, "/index.html" = _RelUrl, _, _, _}} -> - B = - "<HTML><BODY>" ++ - "...some body part..." ++ - "</BODY></HTML>", - Len = integer_to_list(length(B)), - "HTTP/1.1 200 ok\r\n" ++ - "Content-Length:" ++ Len ++ "\r\n\r\n" ++ B - end, - gen_tcp:send(Sock, Msg), - [?HTTP_MAX_HEADER_SIZE]. - + {ok, {{_,503, _}, [_ | _], [_|_]}} = + httpc:request(get, {URL503, []}, [], []), + ets:delete(unavailable). %%------------------------------------------------------------------------- -options(doc) -> - ["Test the option parameters."]; -options(suite) -> +invalid_http(doc) -> + ["Test parse error"]; +invalid_http(suite) -> []; -options(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - {ok, {{_,200,_}, [_ | _], Bin}} - = httpc:request(get, {URL, []}, [{foo, bar}], - %% Ignore unknown options - [{body_format, binary}, {foo, bar}]), - - true = is_binary(Bin), - {ok, {200, [_|_]}} - = httpc:request(get, {URL, []}, [{timeout, infinity}], - [{full_result, false}]); - _ -> - skip("Failed to start local http-server") - end. - +invalid_http(Config) when is_list(Config) -> -%%------------------------------------------------------------------------- + URL = url(group_name(Config), "/invalid_http.html", Config), -http_invalid_http(doc) -> - ["Test parse error"]; -http_invalid_http(suite) -> - []; -http_invalid_http(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/invalid_http.html", - {error, {could_not_parse_as_http, _} = Reason} = httpc:request(get, {URL, []}, [], []), - - test_server:format("Parse error: ~p ~n", [Reason]), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + ct:print("Parse error: ~p ~n", [Reason]). %%------------------------------------------------------------------------- - --define(GOOGLE, "www.google.com"). - -hexed_query_otp_6191(doc) -> - []; -hexed_query_otp_6191(suite) -> - []; -hexed_query_otp_6191(Config) when is_list(Config) -> - Google = ?GOOGLE, - GoogleSearch = "http://" ++ Google ++ "/search", - Search1 = "?hl=en&q=a%D1%85%D1%83%D0%B9&btnG=Google+Search", - URI1 = GoogleSearch ++ Search1, - Search2 = "?hl=en&q=%25%25", - URI2 = GoogleSearch ++ Search2, - Search3 = "?hl=en&q=%foo", - URI3 = GoogleSearch ++ Search3, - - Verify1 = - fun({http, [], ?GOOGLE, 80, "/search", _}) -> ok; - (_) -> error - end, - Verify2 = Verify1, - Verify3 = Verify1, - verify_uri(URI1, Verify1), - verify_uri(URI2, Verify2), - verify_uri(URI3, Verify3), - ok. - -verify_uri(URI, Verify) -> - case http_uri:parse(URI) of - {ok, ParsedURI} -> - case Verify(ParsedURI) of - ok -> - ok; - error -> - Reason = {unexpected_parse_result, URI, ParsedURI}, - ERROR = {error, Reason}, - throw(ERROR) - end; - {error, _} = ERROR -> - throw(ERROR) - end. - - -%%------------------------------------------------------------------------- - -empty_body_otp_6243(doc) -> - ["An empty body was not returned directly. There was a delay for several" - "seconds."]; -empty_body_otp_6243(suite) -> - []; -empty_body_otp_6243(Config) when is_list(Config) -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/empty.html", - {ok, {{_,200,_}, [_ | _], []}} = - httpc:request(get, {URL, []}, [{timeout, 500}], []). - +emulate_lower_versions(doc) -> + [{doc, "Perform request as 0.9 and 1.0 clients."}]; +emulate_lower_versions(Config) when is_list(Config) -> + + URL = url(group_name(Config), "/dummy.html", Config), + + {ok, Body0} = + httpc:request(get, {URL, []}, [{version, "HTTP/0.9"}], []), + inets_test_lib:check_body(Body0), + {ok, {{"HTTP/1.0", 200, _}, [_ | _], Body1 = [_ | _]}} = + httpc:request(get, {URL, []}, [{version, "HTTP/1.0"}], []), + inets_test_lib:check_body(Body1), + {ok, {{"HTTP/1.1", 200, _}, [_ | _], Body2 = [_ | _]}} = + httpc:request(get, {URL, []}, [{version, "HTTP/1.1"}], []), + inets_test_lib:check_body(Body2). %%------------------------------------------------------------------------- -transfer_encoding_otp_6807(doc) -> - ["Transfer encoding is case insensitive"]; -transfer_encoding_otp_6807(suite) -> - []; -transfer_encoding_otp_6807(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ - "/capital_transfer_encoding.html", - {ok, {{_,200,_}, [_|_], [_ | _]}} = httpc:request(URL), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. +relaxed(doc) -> + ["Test relaxed mode"]; +relaxed(Config) when is_list(Config) -> + URL = url(group_name(Config), "/missing_reason_phrase.html", Config), -%%------------------------------------------------------------------------- + {error, Reason} = + httpc:request(get, {URL, []}, [{relaxed, false}], []), -empty_response_header_otp_6830(doc) -> - ["Test the case that the HTTP server does not send any headers"]; -empty_response_header_otp_6830(suite) -> - []; -empty_response_header_otp_6830(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/no_headers.html", - {ok, {{_,200,_}, [], [_ | _]}} = httpc:request(URL), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + ct:print("Not relaxed: ~p~n", [Reason]), + {ok, {{_, 200, _}, [_ | _], [_ | _]}} = + httpc:request(get, {URL, []}, [{relaxed, true}], []). %%------------------------------------------------------------------------- -no_content_204_otp_6982(doc) -> - ["Test the case that the HTTP 204 no content header"]; -no_content_204_otp_6982(suite) -> - []; -no_content_204_otp_6982(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/no_content.html", - {ok, {{_,204,_}, [], []}} = httpc:request(URL), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. - - -%%------------------------------------------------------------------------- +headers() -> + [{doc,"Use as many request headers as possible not used in proxy_headers"}]. +headers(Config) when is_list(Config) -> -missing_CR_otp_7304(doc) -> - ["Test the case that the HTTP server uses only LF instead of CRLF" - "as delimitor"]; -missing_CR_otp_7304(suite) -> - []; -missing_CR_otp_7304(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/missing_CR.html", - {ok, {{_,200,_}, _, [_ | _]}} = httpc:request(URL), - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + URL = url(group_name(Config), "/dummy.html", Config), + DocRoot = ?config(doc_root, Config), + {ok, FileInfo} = + file:read_file_info(filename:join([DocRoot,"dummy.html"])), + CreatedSec = + calendar:datetime_to_gregorian_seconds( + FileInfo#file_info.mtime), -%%------------------------------------------------------------------------- + Mod = httpd_util:rfc1123_date( + calendar:gregorian_seconds_to_datetime( + CreatedSec-1)), + Date = httpd_util:rfc1123_date({date(), time()}), -otp_7883_1(doc) -> - ["OTP-7883-sync"]; -otp_7883_1(suite) -> - []; -otp_7883_1(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), - - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/just_close.html", - {error, socket_closed_remotely} = httpc:request(URL), - DummyServerPid ! stop, - - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(get, {URL, [{"If-Modified-Since", + Mod}, + {"From","[email protected]"}, + {"Date", Date} + ]}, [], []), -otp_7883_2(doc) -> - ["OTP-7883-async"]; -otp_7883_2(suite) -> - []; -otp_7883_2(Config) when is_list(Config) -> - ok = httpc:set_options([{ipfamily, inet}]), + Mod1 = httpd_util:rfc1123_date( + calendar:gregorian_seconds_to_datetime( + CreatedSec+1)), - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ "/just_close.html", - Method = get, - Request = {URL, []}, - HttpOptions = [], - Options = [{sync, false}], - Profile = httpc:default_profile(), - {ok, RequestId} = - httpc:request(Method, Request, HttpOptions, Options, Profile), - ok = - receive - {http, {RequestId, {error, socket_closed_remotely}}} -> - ok - end, - DummyServerPid ! stop, + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(get, {URL, [{"If-UnModified-Since", + Mod1} + ]}, [], []), - ok = httpc:set_options([{ipfamily, inet6fb4}]), - ok. + Tag = httpd_util:create_etag(FileInfo), + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(get, {URL, [{"If-Match", + Tag} + ]}, [], []), + + {ok, {{_,200,_}, [_ | _], _}} = + httpc:request(get, {URL, [{"If-None-Match", + "NotEtag,NeihterEtag"}, + {"Connection", "Close"} + ]}, [], []). %%------------------------------------------------------------------------- +headers_dummy() -> + ["Test the code for handling headers we do not want/can send " + "to a real server. Note it is not logical to send" + "all of these headers together, we only want to test that" + "the code for handling headers will not crash."]. +headers_dummy(Config) when is_list(Config) -> -otp_8154_1(doc) -> - ["OTP-8154"]; -otp_8154_1(suite) -> - []; -otp_8154_1(Config) when is_list(Config) -> - start_inets(), - ReqSeqNumServer = start_sequence_number_server(), - RespSeqNumServer = start_sequence_number_server(), - {ok, Server, Port} = start_slow_server(RespSeqNumServer), - Clients = run_clients(105, Port, ReqSeqNumServer), - %% ok = wait_for_clients(Clients), - ok = wait4clients(Clients, timer:minutes(3)), - Server ! shutdown, - RespSeqNumServer ! shutdown, - ReqSeqNumServer ! shutdown, - ok. - -start_inets() -> - inets:start(), - ok. - - -%% ----------------------------------------------------- -%% A sequence number handler -%% The purpose is to be able to pair requests with responses. - -start_sequence_number_server() -> - proc_lib:spawn(fun() -> loop_sequence_number(1) end). - -loop_sequence_number(N) -> - receive - shutdown -> - ok; - {From, get_next} -> - From ! {next_is, N}, - loop_sequence_number(N + 1) - end. - -get_next_sequence_number(SeqNumServer) -> - SeqNumServer ! {self(), get_next}, - receive {next_is, N} -> N end. - -%% ----------------------------------------------------- -%% Client part -%% Sends requests randomly parallel - -run_clients(NumClients, ServerPort, SeqNumServer) -> - io:format("start clients when" - "~n NumClients: ~w" - "~n ServerPort: ~w" - "~n SeqNumServer: ~w" - "~n", [NumClients, ServerPort, SeqNumServer]), - set_random_seed(), - lists:map( - fun(Id) -> - io:format("starting client ~w~n", [Id]), - Req = f("req~3..0w", [get_next_sequence_number(SeqNumServer)]), - Url = f(?URL_START ++ "~w/~s", [ServerPort, Req]), - Pid = proc_lib:spawn( - fun() -> - io:format("[~w] client started - " - "issue request~n", [Id]), - case httpc:request(Url) of - {ok, {{_,200,_}, _, Resp}} -> - io:format("[~w] 200 response: " - "~p~n", [Id, Resp]), - case lists:prefix(Req++"->", Resp) of - true -> exit(normal); - false -> exit({bad_resp,Req,Resp}) - end; - {ok, {{_,EC,Reason},_,Resp}} -> - io:format("[~w] ~w response: " - "~s~n~s~n", - [Id, EC, Reason, Resp]), - exit({bad_resp,Req,Resp}); - Crap -> - io:format("[~w] bad response: ~p", - [Id, Crap]), - exit({bad_resp, Req, Crap}) - end - end), - MRef = erlang:monitor(process, Pid), - timer:sleep(10 + random:uniform(1334)), - {Id, Pid, MRef} - - end, - lists:seq(1, NumClients)). - -%% wait_for_clients(Clients) -> -%% lists:foreach( -%% fun({Id, Pid, MRef}) -> -%% io:format("waiting for client ~w termination~n", [Id]), -%% receive -%% {'DOWN', MRef, process, Pid, normal} -> -%% io:format("waiting for clients: " -%% "normal exit from ~w (~p)~n", -%% [Id, Pid]), -%% ok; -%% {'DOWN', MRef, process, Pid, Reason} -> -%% io:format("waiting for clients: " -%% "unexpected exit from ~w (~p):" -%% "~n Reason: ~p" -%% "~n", [Id, Pid, Reason]), -%% erlang:error(Reason) -%% end -%% end, -%% Clients). - - -wait4clients([], _Timeout) -> - ok; -wait4clients(Clients, Timeout) when Timeout > 0 -> - io:format("wait4clients -> entry with" - "~n length(Clients): ~w" - "~n Timeout: ~w" - "~n", [length(Clients), Timeout]), - T = t(), - receive - {'DOWN', _MRef, process, Pid, normal} -> - case lists:keysearch(Pid, 2, Clients) of - {value, {Id, _, _}} -> - io:format("receive normal exit message " - "from client ~p (~p)", [Id, Pid]), - NewClients = - lists:keydelete(Id, 1, Clients), - wait4clients(NewClients, - Timeout - (t() - T)); - false -> - io:format("receive normal exit message " - "from unknown process: ~p", [Pid]), - wait4clients(Clients, Timeout - (t() - T)) - end; - - {'DOWN', _MRef, process, Pid, Reason} -> - case lists:keysearch(Pid, 2, Clients) of - {value, {Id, _, _}} -> - io:format("receive bad exit message " - "from client ~p (~p):" - "~n ~p", [Id, Pid, Reason]), - erlang:error({bad_client_termination, Id, Reason}); - false -> - io:format("receive normal exit message " - "from unknown process: ~p", [Pid]), - wait4clients(Clients, Timeout - (t() - T)) - end - - after Timeout -> - erlang:error({client_timeout, Clients}) - end; -wait4clients(Clients, _) -> - erlang:error({client_timeout, Clients}). - - -%% Time in milli seconds -t() -> - {A,B,C} = erlang:now(), - A*1000000000+B*1000+(C div 1000). + URL = url(group_name(Config), "/dummy_headers.html", Config), + Foo = http_chunk:encode("foobar") ++ + binary_to_list(http_chunk:encode_last()), + FooBar = Foo ++ "\r\n\r\nOther:inets_test\r\n\r\n", -%% ----------------------------------------------------- -%% Webserver part: -%% Implements a web server that sends responses one character -%% at a time, with random delays between the characters. + UserPasswd = base64:encode_to_string("Alladin:Sesame"), + Auth = "Basic " ++ UserPasswd, -start_slow_server(SeqNumServer) -> - io:format("start slow server when" - "~n SeqNumServer: ~w" - "~n", [SeqNumServer]), - proc_lib:start( - erlang, apply, [fun() -> init_slow_server(SeqNumServer) end, []]). + %% The dummy server will ignore the headers, we only want to test + %% that the client header-handling code. This would not + %% be a vaild http-request! + {ok, {{_,200,_}, [_ | _], [_|_]}} = + httpc:request(post, + {URL, + [{"Via", + "1.0 fred, 1.1 nowhere.com (Apache/1.1)"}, + {"Warning","1#pseudonym foobar"}, + {"Vary","*"}, + {"Upgrade","HTTP/2.0"}, + {"Pragma", "1#no-cache"}, + {"Cache-Control", "no-cache"}, + {"Connection", "close"}, + {"Date", "Sat, 29 Oct 1994 19:43:31 GMT"}, + {"Accept", " text/plain; q=0.5, text/html"}, + {"Accept-Language", "en"}, + {"Accept-Encoding","chunked"}, + {"Accept-Charset", "ISO8859-1"}, + {"Authorization", Auth}, + {"Expect", "1#100-continue"}, + {"User-Agent","inets"}, + {"Transfer-Encoding","chunked"}, + {"Range", " bytes=0-499"}, + {"If-Range", "Sat, 29 Oct 1994 19:43:31 GMT"}, + {"If-Match", "*"}, + {"Content-Type", "text/plain"}, + {"Content-Encoding", "chunked"}, + {"Content-Length", "6"}, + {"Content-Language", "en"}, + {"Content-Location", "http://www.foobar.se"}, + {"Content-MD5", + "104528739076276072743283077410617235478"}, + {"Content-Range", "bytes 0-499/1234"}, + {"Allow", "GET"}, + {"Proxy-Authorization", Auth}, + {"Expires", "Sat, 29 Oct 1994 19:43:31 GMT"}, + {"Upgrade", "HTTP/2.0"}, + {"Last-Modified", "Sat, 29 Oct 1994 19:43:31 GMT"}, + {"Trailer","1#User-Agent"} + ], "text/plain", FooBar}, + [], []). -init_slow_server(SeqNumServer) -> - io:format("[webserver ~w] init slow server" - "~n", [SeqNumServer]), - {ok, LSock} = gen_tcp:listen(0, [binary, {packet,0}, {active,true}, - {backlog, 100}]), - io:format("[webserver ~w] LSock: ~p" - "~n", [SeqNumServer, LSock]), - {ok, {_IP, Port}} = inet:sockname(LSock), - io:format("[webserver ~w] Port: ~w" - "~n", [SeqNumServer, Port]), - proc_lib:init_ack({ok, self(), Port}), - loop_slow_server(LSock, SeqNumServer). -loop_slow_server(LSock, SeqNumServer) -> - io:format("[webserver ~w] entry with" - "~n LSock: ~p" - "~n", [SeqNumServer, LSock]), - Master = self(), - Acceptor = proc_lib:spawn( - fun() -> client_handler(Master, LSock, SeqNumServer) end), - io:format("[webserver ~w] acceptor started" - "~n Acceptor: ~p" - "~n", [SeqNumServer, Acceptor]), - receive - {accepted, Acceptor} -> - io:format("[webserver ~w] accepted" - "~n", [SeqNumServer]), - loop_slow_server(LSock, SeqNumServer); - shutdown -> - gen_tcp:close(LSock), - exit(Acceptor, kill) - end. +%%------------------------------------------------------------------------- +remote_socket_close(Config) when is_list(Config) -> + URL = url(group_name(Config), "/just_close.html", Config), + {error, socket_closed_remotely} = httpc:request(URL). -%% Handle one client connection -client_handler(Master, LSock, SeqNumServer) -> - io:format("[acceptor ~w] await accept" - "~n", [SeqNumServer]), - {ok, CSock} = gen_tcp:accept(LSock), - io:format("[acceptor ~w] accepted" - "~n CSock: ~p" - "~n", [SeqNumServer, CSock]), - Master ! {accepted, self()}, - set_random_seed(), - loop_client(1, CSock, SeqNumServer). +%%------------------------------------------------------------------------- -loop_client(N, CSock, SeqNumServer) -> - %% Await request, don't bother parsing it too much, - %% assuming the entire request arrives in one packet. - io:format("[acceptor ~w] await request" - "~n N: ~p" - "~n", [SeqNumServer, N]), +remote_socket_close_async(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/just_close.html", Config), []}, + Options = [{sync, false}], + Profile = httpc:default_profile(), + {ok, RequestId} = + httpc:request(get, Request, [], Options, Profile), receive - {tcp, CSock, Req} -> - ReqNum = parse_req_num(Req), - RespSeqNum = get_next_sequence_number(SeqNumServer), - Response = f("~s->resp~3..0w/~2..0w", [ReqNum, RespSeqNum, N]), - Txt = f("Slow server (~p) got ~p, answering with ~p", - [self(), Req, Response]), - io:format("~s...~n", [Txt]), - slowly_send_response(CSock, Response), - case parse_connection_type(Req) of - keep_alive -> - io:format("~s...done~n", [Txt]), - loop_client(N+1, CSock, SeqNumServer); - close -> - io:format("~s...done (closing)~n", [Txt]), - gen_tcp:close(CSock) - end + {http, {RequestId, {error, socket_closed_remotely}}} -> + ok end. -slowly_send_response(CSock, Answer) -> - Response = f("HTTP/1.1 200 OK\r\nContent-Length: ~w\r\n\r\n~s", - [length(Answer), Answer]), - lists:foreach( - fun(Char) -> - timer:sleep(random:uniform(500)), - gen_tcp:send(CSock, <<Char>>) - end, - Response). +%%------------------------------------------------------------------------- -parse_req_num(Request) -> - Opts = [caseless,{capture,all_but_first,list}], - {match, [ReqNum]} = re:run(Request, "GET /(.*) HTTP", Opts), - ReqNum. +stream_to_pid(Config) when is_list(Config) -> + ReceiverPid = create_receiver(pid), + Receiver = ReceiverPid, -parse_connection_type(Request) -> - Opts = [caseless,{capture,all_but_first,list}], - {match,[CType]} = re:run(Request, "connection: *(keep-alive|close)", Opts), - case string:to_lower(CType) of - "close" -> close; - "keep-alive" -> keep_alive - end. + stream(ReceiverPid, Receiver, Config), + stop_receiver(ReceiverPid). -set_random_seed() -> - {_, _, Micros} = now(), - A = erlang:phash2([make_ref(), self(), Micros]), - random:seed(A, A, A). +stream_through_fun(Config) when is_list(Config) -> + ReceiverPid = create_receiver(function), + Receiver = stream_deliver_fun(ReceiverPid), -f(F, A) -> lists:flatten(io_lib:format(F,A)). + stream(ReceiverPid, Receiver, Config), + stop_receiver(ReceiverPid). +stream_through_mfa(Config) when is_list(Config) -> + ReceiverPid = create_receiver(mfa), + Receiver = {?MODULE, stream_deliver, [mfa, ReceiverPid]}, + stream(ReceiverPid, Receiver, Config). %%------------------------------------------------------------------------- +inet_opts(Config) when is_list(Config) -> + MaxSessions = 5, + MaxKeepAlive = 10, + KeepAliveTimeout = timer:minutes(2), + ConnOptions = [{max_sessions, MaxSessions}, + {max_keep_alive_length, MaxKeepAlive}, + {keep_alive_timeout, KeepAliveTimeout}], + httpc:set_options(ConnOptions), + + Request = {url(group_name(Config), "/dummy.html", Config), []}, + Timeout = timer:seconds(1), + ConnTimeout = Timeout + timer:seconds(1), + HttpOptions = [{timeout, Timeout}, {connect_timeout, ConnTimeout}], + Options0 = [{socket_opts, [{tos, 87}, + {recbuf, 16#FFFF}, + {sndbuf, 16#FFFF}]}], + + {ok, {{_,200,_}, [_ | _], ReplyBody0 = [_ | _]}} = httpc:request(get, Request, HttpOptions, Options0), + inets_test_lib:check_body(ReplyBody0), + + Options1 = [{socket_opts, [{tos, 84}, + {recbuf, 32#1FFFF}, + {sndbuf, 32#1FFFF}]}], + {ok, {{_,200,_}, [_ | _], ReplyBody1 = [_ | _]}} = httpc:request(get, Request, [], Options1), + inets_test_lib:check_body(ReplyBody1). +%%------------------------------------------------------------------------- +port_in_host_header(Config) when is_list(Config) -> -otp_8106_pid(doc) -> - ["OTP-8106 - deliver reply info using \"other\" pid"]; -otp_8106_pid(suite) -> - []; -otp_8106_pid(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - ReceiverPid = create_receiver(pid), - Receiver = ReceiverPid, - - otp8106(ReceiverPid, Receiver, Config), - - stop_receiver(ReceiverPid), - - ok; - _ -> - skip("Failed to start local http-server") - end. - - -otp_8106_fun(doc) -> - ["OTP-8106 - deliver reply info using fun"]; -otp_8106_fun(suite) -> - []; -otp_8106_fun(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - ReceiverPid = create_receiver(function), - Receiver = otp_8106_deliver_fun(ReceiverPid), - - otp8106(ReceiverPid, Receiver, Config), + Request = {url(group_name(Config), "/ensure_host_header_with_port.html", Config), []}, + {ok, {{_, 200, _}, _, Body}} = httpc:request(get, Request, [], []), + inets_test_lib:check_body(Body). - stop_receiver(ReceiverPid), - - ok; - _ -> - skip("Failed to start local http-server") +%%------------------------------------------------------------------------- +timeout_memory_leak() -> + [{doc, "Check OTP-8739"}]. +timeout_memory_leak(Config) when is_list(Config) -> + {_DummyServerPid, Port} = otp_8739_dummy_server(), + {ok,Host} = inet:gethostname(), + Request = {?URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ "/dummy.html", []}, + case httpc:request(get, Request, [{connect_timeout, 500}, {timeout, 1}], [{sync, true}]) of + {error, timeout} -> + %% And now we check the size of the handler db + Info = httpc:info(), + ct:print("Info: ~p", [Info]), + {value, {handlers, Handlers}} = + lists:keysearch(handlers, 1, Info), + case Handlers of + [] -> + ok; + _ -> + ct:fail({unexpected_handlers, Handlers}) + end; + Unexpected -> + ct:fail({unexpected, Unexpected}) end. +%%-------------------------------------------------------------------- -otp_8106_mfa(doc) -> - ["OTP-8106 - deliver reply info using mfa callback"]; -otp_8106_mfa(suite) -> - []; -otp_8106_mfa(Config) when is_list(Config) -> - case ?config(local_server, Config) of - ok -> - ReceiverPid = create_receiver(mfa), - Receiver = {?MODULE, otp_8106_deliver, [mfa, ReceiverPid]}, - - otp8106(ReceiverPid, Receiver, Config), - - stop_receiver(ReceiverPid), - - ok; - _ -> - skip("Failed to start local http-server") - end. - - - otp8106(ReceiverPid, Receiver, Config) -> - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - Request = {URL, []}, - HTTPOptions = [], - Options = [{sync, false}, {receiver, Receiver}], +wait_for_whole_response() -> + [{doc, "Check OTP-8154"}]. +wait_for_whole_response(Config) when is_list(Config) -> - {ok, RequestId} = - httpc:request(get, Request, HTTPOptions, Options), + ReqSeqNumServer = start_sequence_number_server(), + RespSeqNumServer = start_sequence_number_server(), + {ok, Server, Port} = start_slow_server(RespSeqNumServer), + Clients = run_clients(105, Port, ReqSeqNumServer), + ok = wait4clients(Clients, timer:minutes(3)), + Server ! shutdown, + RespSeqNumServer ! shutdown, + ReqSeqNumServer ! shutdown. - Body = - receive +%%-------------------------------------------------------------------- +%% Internal Functions ------------------------------------------------ +%%-------------------------------------------------------------------- +stream(ReceiverPid, Receiver, Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + Options = [{sync, false}, {receiver, Receiver}], + {ok, RequestId} = + httpc:request(get, Request, [], Options), + Body = + receive {reply, ReceiverPid, {RequestId, {{_, 200, _}, _, B}}} -> B; {reply, ReceiverPid, Msg} -> - tsf(Msg); + ct:fail(Msg); {bad_reply, ReceiverPid, Msg} -> - tsf(Msg) + ct:fail(Msg) end, - inets_test_lib:check_body(binary_to_list(Body)), - ok. - + inets_test_lib:check_body(binary_to_list(Body)). create_receiver(Type) -> - Parent = self(), + Parent = self(), Receiver = fun() -> receiver(Type, Parent) end, spawn_link(Receiver). @@ -2678,8 +941,7 @@ stop_receiver(Pid) -> receiver(Type, Parent) -> receive {stop, Parent} -> - exit(normal); - + ok; {http, ReplyInfo} when (Type =:= pid) -> Parent ! {reply, self(), ReplyInfo}, receiver(Type, Parent); @@ -2687,258 +949,116 @@ receiver(Type, Parent) -> {Type, ReplyInfo} -> Parent ! {reply, self(), ReplyInfo}, receiver(Type, Parent); - + Crap -> Parent ! {reply, self(), {bad_reply, Crap}}, receiver(Type, Parent) end. +stream_deliver_fun(ReceiverPid) -> + fun(ReplyInfo) -> stream_deliver(ReplyInfo, function, ReceiverPid) end. -otp_8106_deliver_fun(ReceiverPid) -> - fun(ReplyInfo) -> otp_8106_deliver(ReplyInfo, function, ReceiverPid) end. - -otp_8106_deliver(ReplyInfo, Type, ReceiverPid) -> +stream_deliver(ReplyInfo, Type, ReceiverPid) -> ReceiverPid ! {Type, ReplyInfo}, ok. +stream_test(Request, To) -> + {ok, {{_,200,_}, [_ | _], Body}} = + httpc:request(get, Request, [], []), + {ok, RequestId} = + httpc:request(get, Request, [], [{sync, false}, To]), + StreamedBody = + receive + {http, {RequestId, stream_start, _Headers}} -> + receive_streamed_body(RequestId, <<>>); + {http, {RequestId, stream_start, _Headers, Pid}} -> + receive_streamed_body(RequestId, <<>>, Pid); + {http, Msg} -> + ct:fail(Msg) + end, -%%------------------------------------------------------------------------- - -otp_8056(doc) -> - "OTP-8056"; -otp_8056(suite) -> - []; -otp_8056(Config) when is_list(Config) -> - Method = get, - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - Request = {URL, []}, - HTTPOptions = [], - Options1 = [{sync, true}, {stream, {self, once}}], - Options2 = [{sync, true}, {stream, self}], - {error, streaming_error} = httpc:request(Method, Request, - HTTPOptions, Options1), - tsp("request 1 failed as expected"), - {error, streaming_error} = httpc:request(Method, Request, - HTTPOptions, Options2), - tsp("request 2 failed as expected"), - ok. - - -%%------------------------------------------------------------------------- - -otp_8352(doc) -> - "OTP-8352"; -otp_8352(suite) -> - []; -otp_8352(Config) when is_list(Config) -> - tsp("otp_8352 -> entry with" - "~n Config: ~p", [Config]), - case ?config(local_server, Config) of - ok -> - tsp("local-server running"), - - tsp("initial profile info(1): ~p", [httpc:info()]), - - MaxSessions = 5, - MaxKeepAlive = 10, - KeepAliveTimeout = timer:minutes(2), - ConnOptions = [{max_sessions, MaxSessions}, - {max_keep_alive_length, MaxKeepAlive}, - {keep_alive_timeout, KeepAliveTimeout}], - httpc:set_options(ConnOptions), - - Method = get, - Port = ?config(local_port, Config), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - Request = {URL, []}, - Timeout = timer:seconds(1), - ConnTimeout = Timeout + timer:seconds(1), - HttpOptions1 = [{timeout, Timeout}, {connect_timeout, ConnTimeout}], - Options1 = [{socket_opts, [{tos, 87}, - {recbuf, 16#FFFF}, - {sndbuf, 16#FFFF}]}], - case httpc:request(Method, Request, HttpOptions1, Options1) of - {ok, {{_,200,_}, [_ | _], ReplyBody1 = [_ | _]}} -> - %% equivaliant to httpc:request(get, {URL, []}, [], []), - inets_test_lib:check_body(ReplyBody1); - {ok, UnexpectedReply1} -> - tsf({unexpected_reply, UnexpectedReply1}); - {error, _} = Error1 -> - tsf({bad_reply, Error1}) - end, - - tsp("profile info (2): ~p", [httpc:info()]), - - HttpOptions2 = [], - Options2 = [{socket_opts, [{tos, 84}, - {recbuf, 32#1FFFF}, - {sndbuf, 32#1FFFF}]}], - case httpc:request(Method, Request, HttpOptions2, Options2) of - {ok, {{_,200,_}, [_ | _], ReplyBody2 = [_ | _]}} -> - %% equivaliant to httpc:request(get, {URL, []}, [], []), - inets_test_lib:check_body(ReplyBody2); - {ok, UnexpectedReply2} -> - tsf({unexpected_reply, UnexpectedReply2}); - {error, _} = Error2 -> - tsf({bad_reply, Error2}) - end, - tsp("profile info (3): ~p", [httpc:info()]), - ok; - - _ -> - skip("Failed to start local http-server") - end. - - -%%------------------------------------------------------------------------- - -otp_8371(doc) -> - ["OTP-8371"]; -otp_8371(suite) -> - []; -otp_8371(Config) when is_list(Config) -> - ok = httpc:set_options([{ipv6, disabled}]), % also test the old option - {DummyServerPid, Port} = dummy_server(ipv4), - - URL = ?URL_START ++ integer_to_list(Port) ++ - "/ensure_host_header_with_port.html", - - case httpc:request(get, {URL, []}, [], []) of - {ok, Result} -> - case Result of - {{_, 200, _}, _Headers, Body} -> - tsp("expected response with" - "~n Body: ~p", [Body]), - ok; - {StatusLine, Headers, Body} -> - tsp("expected response with" - "~n StatusLine: ~p" - "~n Headers: ~p" - "~n Body: ~p", [StatusLine, Headers, Body]), - tsf({unexpected_result, - [{status_line, StatusLine}, - {headers, Headers}, - {body, Body}]}); - _ -> - tsf({unexpected_result, Result}) - end; - Error -> - tsf({request_failed, Error}) - end, + Body == binary_to_list(StreamedBody). - DummyServerPid ! stop, - ok = httpc:set_options([{ipv6, enabled}]), +url(http, End, Config) -> + Port = ?config(port, Config), + {ok,Host} = inet:gethostname(), + ?URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End; +url(https, End, Config) -> + Port = ?config(port, Config), + {ok,Host} = inet:gethostname(), + ?TLS_URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End; +url(sim_http, End, Config) -> + url(http, End, Config); +url(sim_https, End, Config) -> + url(https, End, Config). +url(http, UserInfo, End, Config) -> + Port = ?config(port, Config), + ?URL_START ++ UserInfo ++ integer_to_list(Port) ++ End; +url(https, UserInfo, End, Config) -> + Port = ?config(port, Config), + ?TLS_URL_START ++ UserInfo ++ integer_to_list(Port) ++ End; +url(sim_http, UserInfo, End, Config) -> + url(http, UserInfo, End, Config); +url(sim_https, UserInfo, End, Config) -> + url(https, UserInfo, End, Config). + +group_name(Config) -> + GroupProp = ?config(tc_group_properties, Config), + proplists:get_value(name, GroupProp). + +server_start(sim_http, _) -> + Inet = inet_version(), + ok = httpc:set_options([{ipfamily, Inet}]), + {_Pid, Port} = dummy_server(Inet), + Port; + +server_start(sim_https, SslConfig) -> + Inet = inet_version(), + ok = httpc:set_options([{ipfamily, Inet}]), + {_Pid, Port} = dummy_server(ssl, Inet, SslConfig), + Port; + +server_start(_, HttpdConfig) -> + {ok, Pid} = inets:start(httpd, HttpdConfig), + Serv = inets:services_info(), + {value, {_, _, Info}} = lists:keysearch(Pid, 2, Serv), + proplists:get_value(port, Info). + +server_config(http, Config) -> + ServerRoot = ?config(server_root, Config), + [{port, 0}, + {server_name,"httpc_test"}, + {server_root, ServerRoot}, + {document_root, ?config(doc_root, Config)}, + {bind_address, any}, + {ipfamily, inet_version()}, + {mime_type, "text/plain"}, + {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} + ]; + +server_config(https, Config) -> + [{socket_type, {essl, ssl_config(Config)}} | server_config(http, Config)]; +server_config(sim_https, Config) -> + ssl_config(Config); +server_config(_, _) -> + []. + +start_apps(https) -> + inets_test_lib:start_apps([crypto, public_key, ssl]); +start_apps(_) -> ok. - -%%------------------------------------------------------------------------- - -otp_8739(doc) -> - ["OTP-8739"]; -otp_8739(suite) -> - []; -otp_8739(Config) when is_list(Config) -> - {_DummyServerPid, Port} = otp_8739_dummy_server(), - URL = ?URL_START ++ integer_to_list(Port) ++ "/dummy.html", - Method = get, - Request = {URL, []}, - HttpOptions = [{connect_timeout, 500}, {timeout, 1}], - Options = [{sync, true}], - case httpc:request(Method, Request, HttpOptions, Options) of - {error, timeout} -> - %% And now we check the size of the handler db - Info = httpc:info(), - tsp("Info: ~p", [Info]), - {value, {handlers, Handlers}} = - lists:keysearch(handlers, 1, Info), - case Handlers of - [] -> - ok; - _ -> - tsf({unexpected_handlers, Handlers}) - end; - Unexpected -> - tsf({unexpected, Unexpected}) - end. - - -otp_8739_dummy_server() -> - Parent = self(), - Pid = spawn_link(fun() -> otp_8739_dummy_server_init(Parent) end), - receive - {port, Port} -> - {Pid, Port} - end. - -otp_8739_dummy_server_init(Parent) -> - {ok, ListenSocket} = - gen_tcp:listen(0, [binary, inet, {packet, 0}, - {reuseaddr,true}, - {active, false}]), - {ok, Port} = inet:port(ListenSocket), - Parent ! {port, Port}, - otp_8739_dummy_server_main(Parent, ListenSocket). - -otp_8739_dummy_server_main(_Parent, ListenSocket) -> - case gen_tcp:accept(ListenSocket) of - {ok, Sock} -> - %% Ignore the request, and simply wait for the socket to close - receive - {tcp_closed, Sock} -> - (catch gen_tcp:close(ListenSocket)), - exit(normal); - {tcp_error, Sock, Reason} -> - tsp("socket error: ~p", [Reason]), - (catch gen_tcp:close(ListenSocket)), - exit(normal) - after 10000 -> - %% Just in case - (catch gen_tcp:close(Sock)), - (catch gen_tcp:close(ListenSocket)), - exit(timeout) - end; - Error -> - exit(Error) - end. - - -%%------------------------------------------------------------------------- - -initial_server_connect(doc) -> - ["If this test cases times out the init of httpc_handler process is" - "blocking the manager/client process (implementation dependent which) but nither" - "should be blocked."]; -initial_server_connect(suite) -> - []; -initial_server_connect(Config) when is_list(Config) -> +ssl_config(Config) -> DataDir = ?config(data_dir, Config), - ok = httpc:set_options([{ipfamily, inet}]), - - CertFile = filename:join(DataDir, "ssl_server_cert.pem"), - SSLOptions = [{certfile, CertFile}, {keyfile, CertFile}], - - {DummyServerPid, Port} = dummy_ssl_server_hang(self(), ipv4, SSLOptions), - - URL = ?SSL_URL_START ++ integer_to_list(Port) ++ "/index.html", - - httpc:request(get, {URL, []}, [{ssl,{essl,[]}}], [{sync, false}]), - - [{session_cookies,[]}] = httpc:which_cookies(), + [{certfile, filename:join(DataDir, "ssl_server_cert.pem")}, + {verify, verify_none} + ]. - DummyServerPid ! stop, - ok = httpc:set_options([{ipfamily, inet6fb4}]). - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- setup_server_dirs(ServerRoot, DocRoot, DataDir) -> - ConfDir = filename:join(ServerRoot, "conf"), CgiDir = filename:join(ServerRoot, "cgi-bin"), ok = file:make_dir(ServerRoot), ok = file:make_dir(DocRoot), - ok = file:make_dir(ConfDir), ok = file:make_dir(CgiDir), {ok, Files} = file:list_dir(DataDir), @@ -2961,77 +1081,41 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> end, inets_test_lib:copy_file(Cgi, DataDir, CgiDir), - inets_test_lib:copy_file("mime.types", DataDir, ConfDir). - -create_config(FileName, ComType, Port, PrivDir, ServerRoot, DocRoot, - SSLDir) -> - MaxHdrSz = io_lib:format("~p", [256]), - MaxHdrAct = io_lib:format("~p", [close]), - SSL = - case ComType of - ssl -> - [cline(["SSLCertificateFile ", - filename:join(SSLDir, "ssl_server_cert.pem")]), - cline(["SSLCertificateKeyFile ", - filename:join(SSLDir, "ssl_server_cert.pem")]), - cline(["SSLVerifyClient 0"])]; - _ -> - [] - end, + AbsCgi = filename:join([CgiDir, Cgi]), + {ok, FileInfo} = file:read_file_info(AbsCgi), + ok = file:write_file_info(AbsCgi, FileInfo#file_info{mode = 8#00755}). - Mod_order = "Modules mod_alias mod_auth mod_esi mod_actions mod_cgi" - " mod_include mod_dir mod_get mod_head" - " mod_log mod_disk_log mod_trace", - - %% BindAddress = "*|inet", % Force the use of IPv4 - BindAddress = "*", % This corresponds to using IpFamily inet6fb4 - - HttpConfig = [ - cline(["BindAddress ", BindAddress]), - cline(["Port ", integer_to_list(Port)]), - cline(["ServerName ", "httpc_test"]), - cline(["SocketType ", atom_to_list(ComType)]), - cline([Mod_order]), - cline(["ServerRoot ", ServerRoot]), - cline(["DocumentRoot ", DocRoot]), - cline(["MaxHeaderSize ",MaxHdrSz]), - cline(["MaxHeaderAction ",MaxHdrAct]), - cline(["DirectoryIndex ", "index.html "]), - cline(["DefaultType ", "text/plain"]), - cline(["ScriptAlias /cgi-bin/ ", - filename:join(ServerRoot, "cgi-bin"), "/"]), - SSL], - ConfigFile = filename:join([PrivDir,FileName]), - {ok, Fd} = file:open(ConfigFile, [write]), - ok = file:write(Fd, lists:flatten(HttpConfig)), - ok = file:close(Fd). - -cline(List) -> - lists:flatten([List, "\r\n"]). -receive_streamed_body(RequestId, Body) -> - receive - {http, {RequestId, stream, BinBodyPart}} -> - receive_streamed_body(RequestId, - <<Body/binary, BinBodyPart/binary>>); - {http, {RequestId, stream_end, _Headers}} -> - Body; - {http, Msg} -> - tsf(Msg) - end. +keep_alive_requests(Request, Profile) -> + {ok, RequestIdA0} = + httpc:request(get, Request, [], [{sync, false}], Profile), + {ok, RequestIdA1} = + httpc:request(get, Request, [], [{sync, false}], Profile), + {ok, RequestIdA2} = + httpc:request(get, Request, [], [{sync, false}], Profile), -receive_streamed_body(RequestId, Body, Pid) -> - httpc:stream_next(Pid), - test_server:format("~p:receive_streamed_body -> requested next stream ~n", [?MODULE]), - receive - {http, {RequestId, stream, BinBodyPart}} -> - receive_streamed_body(RequestId, - <<Body/binary, BinBodyPart/binary>>, - Pid); - {http, {RequestId, stream_end, _Headers}} -> - Body; - {http, Msg} -> - tsf(Msg) + receive_replys([RequestIdA0, RequestIdA1, RequestIdA2]), + + {ok, RequestIdB0} = + httpc:request(get, Request, [], [{sync, false}], Profile), + {ok, RequestIdB1} = + httpc:request(get, Request, [], [{sync, false}], Profile), + {ok, RequestIdB2} = + httpc:request(get, Request, [], [{sync, false}], Profile), + + ok = httpc:cancel_request(RequestIdB1, Profile), + ct:print("Cancel ~p~n", [RequestIdB1]), + receive_replys([RequestIdB0, RequestIdB2]). + + +receive_replys([]) -> + ok; +receive_replys([ID|IDs]) -> + receive + {http, {ID, {{_, 200, _}, [_|_], _}}} -> + receive_replys(IDs); + {http, {Other, {{_, 200, _}, [_|_], _}}} -> + ct:fail({recived_canceld_id, Other}) end. %% Perform a synchronous stop @@ -3042,55 +1126,46 @@ dummy_server_stop(Pid) -> ok end. -dummy_server(IpV) -> - dummy_server(self(), ip_comm, IpV, []). +inet_version() -> + inet. %% Just run inet for now + %% case gen_tcp:listen(0,[inet6]) of + %% {ok, S} -> + %% gen_tcp:close(S), + %% inet6; + %% _ -> + %% inet + %%end. + +dummy_server(Inet) -> + dummy_server(self(), ip_comm, Inet, []). -dummy_server(SocketType, IpV, Extra) -> - dummy_server(self(), SocketType, IpV, Extra). +dummy_server(SocketType, Inet, Extra) -> + dummy_server(self(), SocketType, Inet, Extra). -dummy_server(Caller, SocketType, IpV, Extra) -> - Args = [Caller, SocketType, IpV, Extra], +dummy_server(Caller, SocketType, Inet, Extra) -> + Args = [Caller, SocketType, Inet, Extra], Pid = spawn(httpc_SUITE, dummy_server_init, Args), receive {port, Port} -> {Pid, Port} end. -dummy_server_init(Caller, ip_comm, IpV, _) -> +dummy_server_init(Caller, ip_comm, Inet, _) -> BaseOpts = [binary, {packet, 0}, {reuseaddr,true}, {active, false}], - {ok, ListenSocket} = - case IpV of - ipv4 -> - tsp("ip_comm ipv4 listen", []), - gen_tcp:listen(0, [inet | BaseOpts]); - ipv6 -> - tsp("ip_comm ipv6 listen", []), - gen_tcp:listen(0, [inet6 | BaseOpts]) - end, + {ok, ListenSocket} = gen_tcp:listen(0, [Inet | BaseOpts]), {ok, Port} = inet:port(ListenSocket), - tsp("dummy_server_init(ip_comm) -> Port: ~p", [Port]), Caller ! {port, Port}, dummy_ipcomm_server_loop({httpd_request, parse, [?HTTP_MAX_HEADER_SIZE]}, [], ListenSocket); -dummy_server_init(Caller, essl, IpV, SSLOptions) -> - BaseOpts = [{ssl_imp, new}, - {backlog, 128}, binary, {reuseaddr,true}, {active, false} | + +dummy_server_init(Caller, ssl, Inet, SSLOptions) -> + BaseOpts = [binary, {reuseaddr,true}, {active, false} | SSLOptions], - dummy_ssl_server_init(Caller, BaseOpts, IpV). - -dummy_ssl_server_init(Caller, BaseOpts, IpV) -> - {ok, ListenSocket} = - case IpV of - ipv4 -> - tsp("dummy_ssl_server_init -> ssl ipv4 listen", []), - ssl:listen(0, [inet | BaseOpts]); - ipv6 -> - tsp("dummy_ssl_server_init -> ssl ipv6 listen", []), - ssl:listen(0, [inet6 | BaseOpts]) - end, - tsp("dummy_ssl_server_init -> ListenSocket: ~p", [ListenSocket]), + dummy_ssl_server_init(Caller, BaseOpts, Inet). + +dummy_ssl_server_init(Caller, BaseOpts, Inet) -> + {ok, ListenSocket} = ssl:listen(0, [Inet | BaseOpts]), {ok, {_, Port}} = ssl:sockname(ListenSocket), - tsp("dummy_ssl_server_init -> Port: ~p", [Port]), Caller ! {port, Port}, dummy_ssl_server_loop({httpd_request, parse, [?HTTP_MAX_HEADER_SIZE]}, [], ListenSocket). @@ -3098,85 +1173,56 @@ dummy_ssl_server_init(Caller, BaseOpts, IpV) -> dummy_ipcomm_server_loop(MFA, Handlers, ListenSocket) -> receive stop -> - tsp("dummy_ipcomm_server_loop -> stop handlers", []), lists:foreach(fun(Handler) -> Handler ! stop end, Handlers); {stop, From} -> - tsp("dummy_ipcomm_server_loop -> " - "stop command from ~p for handlers (~p)", [From, Handlers]), Stopper = fun(Handler) -> Handler ! stop end, lists:foreach(Stopper, Handlers), From ! {stopped, self()} after 0 -> - tsp("dummy_ipcomm_server_loop -> await accept", []), {ok, Socket} = gen_tcp:accept(ListenSocket), - tsp("dummy_ipcomm_server_loop -> accepted: ~p", [Socket]), HandlerPid = dummy_request_handler(MFA, Socket), - tsp("dummy_icomm_server_loop -> handler created: ~p", [HandlerPid]), gen_tcp:controlling_process(Socket, HandlerPid), - tsp("dummy_ipcomm_server_loop -> " - "control transfered to handler", []), HandlerPid ! ipcomm_controller, - tsp("dummy_ipcomm_server_loop -> " - "handler informed about control transfer", []), dummy_ipcomm_server_loop(MFA, [HandlerPid | Handlers], - ListenSocket) + ListenSocket) end. dummy_ssl_server_loop(MFA, Handlers, ListenSocket) -> receive stop -> - tsp("dummy_ssl_server_loop -> stop handlers", []), lists:foreach(fun(Handler) -> Handler ! stop end, Handlers); {stop, From} -> - tsp("dummy_ssl_server_loop -> " - "stop command from ~p for handlers (~p)", [From, Handlers]), Stopper = fun(Handler) -> Handler ! stop end, lists:foreach(Stopper, Handlers), From ! {stopped, self()} after 0 -> - tsp("dummy_ssl_server_loop -> await accept", []), {ok, Socket} = ssl:transport_accept(ListenSocket), - tsp("dummy_ssl_server_loop -> accepted: ~p", [Socket]), HandlerPid = dummy_request_handler(MFA, Socket), - tsp("dummy_ssl_server_loop -> handler created: ~p", [HandlerPid]), ssl:controlling_process(Socket, HandlerPid), - tsp("dummy_ssl_server_loop -> control transfered to handler", []), HandlerPid ! ssl_controller, - tsp("dummy_ssl_server_loop -> " - "handler informed about control transfer", []), dummy_ssl_server_loop(MFA, [HandlerPid | Handlers], ListenSocket) end. dummy_request_handler(MFA, Socket) -> - tsp("spawn request handler", []), spawn(httpc_SUITE, dummy_request_handler_init, [MFA, Socket]). dummy_request_handler_init(MFA, Socket) -> SockType = receive ipcomm_controller -> - tsp("dummy_request_handler_init -> " - "received ip_comm controller - activate", []), inet:setopts(Socket, [{active, true}]), ip_comm; ssl_controller -> - tsp("dummy_request_handler_init -> " - "received ssl controller - activate", []), ssl:setopts(Socket, [{active, true}]), ssl end, dummy_request_handler_loop(MFA, SockType, Socket). dummy_request_handler_loop({Module, Function, Args}, SockType, Socket) -> - tsp("dummy_request_handler_loop -> entry with" - "~n Module: ~p" - "~n Function: ~p" - "~n Args: ~p", [Module, Function, Args]), receive {Proto, _, Data} when (Proto =:= tcp) orelse (Proto =:= ssl) -> - tsp("dummy_request_handler_loop -> [~w] Data ~p", [Proto, Data]), - case handle_request(Module, Function, [Data | Args], Socket, Proto) of + case handle_request(Module, Function, [Data | Args], Socket) of stop when Proto =:= tcp -> gen_tcp:close(Socket); stop when Proto =:= ssl -> @@ -3190,49 +1236,26 @@ dummy_request_handler_loop({Module, Function, Args}, SockType, Socket) -> ssl:close(Socket) end. - -mk_close(tcp) -> fun(Sock) -> gen_tcp:close(Sock) end; -mk_close(ssl) -> fun(Sock) -> ssl:close(Sock) end. - -mk_send(tcp) -> fun(Sock, Data) -> gen_tcp:send(Sock, Data) end; -mk_send(ssl) -> fun(Sock, Data) -> ssl:send(Sock, Data) end. - -handle_request(Module, Function, Args, Socket, Proto) -> - Close = mk_close(Proto), - Send = mk_send(Proto), - handle_request(Module, Function, Args, Socket, Close, Send). - -handle_request(Module, Function, Args, Socket, Close, Send) -> - tsp("handle_request -> entry with" - "~n Module: ~p" - "~n Function: ~p" - "~n Args: ~p", [Module, Function, Args]), +handle_request(Module, Function, Args, Socket) -> case Module:Function(Args) of {ok, Result} -> - tsp("handle_request -> ok" - "~n Result: ~p", [Result]), - case (catch handle_http_msg(Result, Socket, Close, Send)) of + case handle_http_msg(Result, Socket) of stop -> stop; <<>> -> - tsp("handle_request -> empty data"), {httpd_request, parse, [[<<>>, ?HTTP_MAX_HEADER_SIZE]]}; Data -> handle_request(httpd_request, parse, - [Data |[?HTTP_MAX_HEADER_SIZE]], Socket, - Close, Send) + [Data |[?HTTP_MAX_HEADER_SIZE]], Socket) end; NewMFA -> - tsp("handle_request -> " - "~n NewMFA: ~p", [NewMFA]), NewMFA end. -handle_http_msg({_, RelUri, _, {_, Headers}, Body}, Socket, Close, Send) -> - tsp("handle_http_msg -> entry with: " - "~n RelUri: ~p" - "~n Headers: ~p" - "~n Body: ~p", [RelUri, Headers, Body]), +handle_http_msg({Method, RelUri, _, {_, Headers}, Body}, Socket) -> + + ct:print("Request: ~p ~p", [Method, RelUri]), + NextRequest = case RelUri of "/dummy_headers.html" -> @@ -3253,225 +1276,69 @@ handle_http_msg({_, RelUri, _, {_, Headers}, Body}, Socket, Close, Send) -> end end, - tsp("handle_http_msg -> NextRequest: ~p", [NextRequest]), case (catch ets:lookup(cookie, cookies)) of [{cookies, true}]-> - tsp("handle_http_msg -> check cookies ~p", []), check_cookie(Headers); _ -> ok end, - + + {ok, {_, Port}} = sockname(Socket), + + DefaultResponse = "HTTP/1.1 200 ok\r\n" ++ "Content-Length:32\r\n\r\n" "<HTML><BODY>foobar</BODY></HTML>", - Msg = - case RelUri of - "/just_close.html" -> - close; - "/no_content.html" -> - "HTTP/1.0 204 No Content\r\n\r\n"; - "/no_headers.html" -> - "HTTP/1.0 200 OK\r\n\r\nTEST"; - "/ensure_host_header_with_port.html" -> - %% tsp("handle_http_msg -> validate host with port"), - case ensure_host_header_with_port(Headers) of - true -> - B = - "<HTML><BODY>" ++ - "host with port" ++ - "</BODY></HTML>", - Len = integer_to_list(length(B)), - "HTTP/1.1 200 ok\r\n" ++ - "Content-Length:" ++ Len ++ "\r\n\r\n" ++ B; - false -> - B = - "<HTML><BODY>" ++ - "Internal Server Error - host without port" ++ - "</BODY></HTML>", - Len = integer_to_list(length(B)), - "HTTP/1.1 500 Internal Server Error\r\n" ++ - "Content-Length:" ++ Len ++ "\r\n\r\n" ++ B - end; - "/300.html" -> - NewUri = ?URL_START ++ - integer_to_list(?IP_PORT) ++ "/dummy.html", - "HTTP/1.1 300 Multiple Choices\r\n" ++ - "Location:" ++ NewUri ++ "\r\n" ++ - "Content-Length:0\r\n\r\n"; - "/301.html" -> - NewUri = ?URL_START ++ - integer_to_list(?IP_PORT) ++ "/dummy.html", - "HTTP/1.1 301 Moved Permanently\r\n" ++ - "Location:" ++ NewUri ++ "\r\n" ++ - "Content-Length:80\r\n\r\n" ++ - "<HTML><BODY><a href=" ++ NewUri ++ - ">New place</a></BODY></HTML>"; - "/302.html" -> - NewUri = ?URL_START ++ - integer_to_list(?IP_PORT) ++ "/dummy.html", - "HTTP/1.1 302 Found \r\n" ++ - "Location:" ++ NewUri ++ "\r\n" ++ - "Content-Length:80\r\n\r\n" ++ - "<HTML><BODY><a href=" ++ NewUri ++ - ">New place</a></BODY></HTML>"; - "/303.html" -> - NewUri = ?URL_START ++ - integer_to_list(?IP_PORT) ++ "/dummy.html", - "HTTP/1.1 303 See Other \r\n" ++ - "Location:" ++ NewUri ++ "\r\n" ++ - "Content-Length:80\r\n\r\n" ++ - "<HTML><BODY><a href=" ++ NewUri ++ - ">New place</a></BODY></HTML>"; - "/307.html" -> - NewUri = ?URL_START ++ - integer_to_list(?IP_PORT) ++ "/dummy.html", - "HTTP/1.1 307 Temporary Rediect \r\n" ++ - "Location:" ++ NewUri ++ "\r\n" ++ - "Content-Length:80\r\n\r\n" ++ - "<HTML><BODY><a href=" ++ NewUri ++ - ">New place</a></BODY></HTML>"; - "/500.html" -> - "HTTP/1.1 500 Internal Server Error\r\n" ++ - "Content-Length:47\r\n\r\n" ++ - "<HTML><BODY>Internal Server Error</BODY></HTML>"; - "/503.html" -> - case ets:lookup(unavailable, 503) of - [{503, unavailable}] -> - ets:insert(unavailable, {503, available}), - "HTTP/1.1 503 Service Unavailable\r\n" ++ - "Retry-After:5\r\n" ++ - "Content-Length:47\r\n\r\n" ++ - "<HTML><BODY>Internal Server Error</BODY></HTML>"; - [{503, available}] -> - DefaultResponse; - [{503, long_unavailable}] -> - "HTTP/1.1 503 Service Unavailable\r\n" ++ - "Retry-After:120\r\n" ++ - "Content-Length:47\r\n\r\n" ++ - "<HTML><BODY>Internal Server Error</BODY></HTML>" - end; - "/redirectloop.html" -> %% Create a potential endless loop! - {ok, Port} = inet:port(Socket), - NewUri = ?URL_START ++ - integer_to_list(Port) ++ "/redirectloop.html", - "HTTP/1.1 300 Multiple Choices\r\n" ++ - "Location:" ++ NewUri ++ "\r\n" ++ - "Content-Length:0\r\n\r\n"; - "/userinfo.html" -> - Challange = "HTTP/1.1 401 Unauthorized \r\n" ++ - "WWW-Authenticate:Basic" ++"\r\n" ++ - "Content-Length:0\r\n\r\n", - case auth_header(Headers) of - {ok, Value} -> - handle_auth(Value, Challange, DefaultResponse); - _ -> - Challange - end; - "/dummy_headers.html" -> - %% The client will only care about the Transfer-Encoding - %% header the rest of these headers are left to the - %% user to evaluate. This is not a valid response - %% it only tests that the header handling code works. - Head = "HTTP/1.1 200 ok\r\n" ++ - "Content-Length:32\r\n" ++ - "Pragma:1#no-cache\r\n" ++ - "Via:1.0 fred, 1.1 nowhere.com (Apache/1.1)\r\n" ++ - "Warning:1#pseudonym foobar\r\n" ++ - "Vary:*\r\n" ++ - "Trailer:Other:inets_test\r\n" ++ - "Upgrade:HTTP/2.0\r\n" ++ - "Age:4711\r\n" ++ - "Transfer-Encoding:chunked\r\n" ++ - "Content-Encoding:foo\r\n" ++ - "Content-Language:en\r\n" ++ - "Content-Location:http://www.foobar.se\r\n" ++ - "Content-MD5:104528739076276072743283077410617235478\r\n" - ++ - "Content-Range:Sat, 29 Oct 1994 19:43:31 GMT\r\n" ++ - "Expires:Sat, 29 Oct 1994 19:43:31 GMT\r\n" ++ - "Proxy-Authenticate:#1Basic" ++ - "\r\n\r\n", - Send(Socket, Head), - Send(Socket, http_chunk:encode("<HTML><BODY>fo")), - Send(Socket, http_chunk:encode("obar</BODY></HTML>")), - http_chunk:encode_last(); - "/capital_transfer_encoding.html" -> - Head = "HTTP/1.1 200 ok\r\n" ++ - "Transfer-Encoding:Chunked\r\n\r\n", - Send(Socket, Head), - Send(Socket, http_chunk:encode("<HTML><BODY>fo")), - Send(Socket, http_chunk:encode("obar</BODY></HTML>")), - http_chunk:encode_last(); - "/cookie.html" -> - "HTTP/1.1 200 ok\r\n" ++ - "set-cookie:" ++ "test_cookie=true; path=/;" ++ - "max-age=60000\r\n" ++ - "Content-Length:32\r\n\r\n"++ - "<HTML><BODY>foobar</BODY></HTML>"; - "/missing_crlf.html" -> - "HTTP/1.1 200 ok" ++ - "Content-Length:32\r\n" ++ - "<HTML><BODY>foobar</BODY></HTML>"; - "/wrong_statusline.html" -> - "ok 200 HTTP/1.1\r\n\r\n" ++ - "Content-Length:32\r\n\r\n" ++ - "<HTML><BODY>foobar</BODY></HTML>"; - "/once_chunked.html" -> - Head = "HTTP/1.1 200 ok\r\n" ++ - "Transfer-Encoding:Chunked\r\n\r\n", - Send(Socket, Head), - Send(Socket, http_chunk:encode("<HTML><BODY>fo")), - Send(Socket, - http_chunk:encode("obar</BODY></HTML>")), - http_chunk:encode_last(); - "/once.html" -> - Head = "HTTP/1.1 200 ok\r\n" ++ - "Content-Length:32\r\n\r\n", - Send(Socket, Head), - Send(Socket, "<HTML><BODY>fo"), - test_server:sleep(1000), - Send(Socket, "ob"), - test_server:sleep(1000), - Send(Socket, "ar</BODY></HTML>"); - "/invalid_http.html" -> - "HTTP/1.1 301\r\nDate:Sun, 09 Dec 2007 13:04:18 GMT\r\n" ++ - "Transfer-Encoding:chunked\r\n\r\n"; - "/missing_reason_phrase.html" -> - "HTTP/1.1 200\r\n" ++ - "Content-Length: 32\r\n\r\n" - "<HTML><BODY>foobar</BODY></HTML>"; - "/missing_CR.html" -> - "HTTP/1.1 200 ok\n" ++ - "Content-Length:32\r\n\n" - "<HTML><BODY>foobar</BODY></HTML>"; - _ -> - DefaultResponse - end, - - tsp("handle_http_msg -> Msg: ~p", [Msg]), + Msg = handle_uri(Method,RelUri, Port, Headers, Socket, DefaultResponse), + case Msg of ok -> - %% Previously, this resulted in an {error, einval}. Now what? ok; close -> %% Nothing to send, just close - Close(Socket); + close(Socket); _ when is_list(Msg) orelse is_binary(Msg) -> - Send(Socket, Msg) + case Msg of + [] -> + ct:print("Empty Msg", []); + _ -> + ct:print("Response: ~p", [Msg]), + send(Socket, Msg) + end end, - tsp("handle_http_msg -> done"), NextRequest. +dummy_ssl_server_hang(Caller, Inet, SslOpt) -> + Pid = spawn(httpc_SUITE, dummy_ssl_server_hang_init, [Caller, Inet, SslOpt]), + receive + {port, Port} -> + {Pid, Port} + end. + +dummy_ssl_server_hang_init(Caller, Inet, SslOpt) -> + {ok, ListenSocket} = + ssl:listen(0, [binary, Inet, {packet, 0}, + {reuseaddr,true}, + {active, false}] ++ SslOpt), + {ok, {_,Port}} = ssl:sockname(ListenSocket), + Caller ! {port, Port}, + {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), + dummy_ssl_server_hang_loop(AcceptSocket). + +dummy_ssl_server_hang_loop(_) -> + %% Do not do ssl:ssl_accept as we + %% want to time out the underlying gen_tcp:connect + receive + stop -> + ok + end. + ensure_host_header_with_port([]) -> false; ensure_host_header_with_port(["host: " ++ Host| _]) -> case string:tokens(Host, [$:]) of - [ActualHost, Port] -> - tsp("ensure_host_header_with_port -> " - "~n ActualHost: ~p" - "~n Port: ~p", [ActualHost, Port]), + [_ActualHost, _Port] -> true; _ -> false @@ -3489,15 +1356,15 @@ auth_header([_ | Tail]) -> handle_auth("Basic " ++ UserInfo, Challange, DefaultResponse) -> case string:tokens(base64:decode_to_string(UserInfo), ":") of ["alladin", "sesame"] = Auth -> - test_server:format("Auth: ~p~n", [Auth]), + ct:print("Auth: ~p~n", [Auth]), DefaultResponse; Other -> - test_server:format("UnAuth: ~p~n", [Other]), + ct:print("UnAuth: ~p~n", [Other]), Challange end. check_cookie([]) -> - tsf(no_cookie_header); + ct:fail(no_cookie_header); check_cookie(["cookie:" ++ _Value | _]) -> ok; check_cookie([_Head | Tail]) -> @@ -3510,86 +1377,532 @@ content_length(["content-length:" ++ Value | _]) -> content_length([_Head | Tail]) -> content_length(Tail). -%% ------------------------------------------------------------------------- - -simple_request_and_verify(Config, - Method, Request, HttpOpts, Opts, VerifyResult) - when (is_list(Config) andalso - is_atom(Method) andalso - is_list(HttpOpts) andalso - is_list(Opts) andalso - is_function(VerifyResult, 1)) -> - tsp("request_and_verify -> entry with" - "~n Method: ~p" - "~n Request: ~p" - "~n HttpOpts: ~p" - "~n Opts: ~p", [Method, Request, HttpOpts, Opts]), - case ?config(local_server, Config) of - ok -> - tsp("request_and_verify -> local-server running"), - Result = (catch httpc:request(Method, Request, HttpOpts, Opts)), - VerifyResult(Result); +handle_uri(_,"/just_close.html",_,_,_,_) -> + close; +handle_uri(_,"/no_content.html",_,_,_,_) -> + "HTTP/1.0 204 No Content\r\n\r\n"; + +handle_uri(_,"/no_headers.html",_,_,_,_) -> + "HTTP/1.0 200 OK\r\n\r\nTEST"; + +handle_uri("TRACE","/trace.html",_,_,_,_) -> + Body = "TRACE /trace.html simulate HTTP TRACE ", + "HTTP/1.1 200 OK\r\n" ++ "Content-Length:" ++ integer_to_list(length(Body)) ++ "\r\n\r\n" ++ Body; + +handle_uri(_,"/ensure_host_header_with_port.html",_,Headers,_,_) -> + case ensure_host_header_with_port(Headers) of + true -> + B = + "<HTML><BODY>" ++ + "host with port" ++ + "</BODY></HTML>", + Len = integer_to_list(length(B)), + "HTTP/1.1 200 ok\r\n" ++ + "Content-Length:" ++ Len ++ "\r\n\r\n" ++ B; + false -> + B = + "<HTML><BODY>" ++ + "Internal Server Error - host without port" ++ + "</BODY></HTML>", + Len = integer_to_list(length(B)), + "HTTP/1.1 500 Internal Server Error\r\n" ++ + "Content-Length:" ++ Len ++ "\r\n\r\n" ++ B + end; + +handle_uri(_,"/300.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 300 Multiple Choices\r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; + +handle_uri("HEAD","/301.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + "HTTP/1.1 301 Moved Permanently\r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:0\r\n\r\n"; + +handle_uri(_,"/301.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 301 Moved Permanently\r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; + +handle_uri("HEAD","/302.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + "HTTP/1.1 302 Found \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:0\r\n\r\n"; + +handle_uri(_,"/302.html",Port, _,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 302 Found \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; + +handle_uri("HEAD","/303.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + "HTTP/1.1 302 See Other \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:0\r\n\r\n"; +handle_uri(_,"/303.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 303 See Other \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; +handle_uri("HEAD","/307.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + "HTTP/1.1 307 Temporary Rediect \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:0\r\n\r\n"; +handle_uri(_,"/307.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/dummy.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 307 Temporary Rediect \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; + +handle_uri(_,"/500.html",_,_,_,_) -> + "HTTP/1.1 500 Internal Server Error\r\n" ++ + "Content-Length:47\r\n\r\n" ++ + "<HTML><BODY>Internal Server Error</BODY></HTML>"; + +handle_uri(_,"/503.html",_,_,_,DefaultResponse) -> + case ets:lookup(unavailable, 503) of + [{503, unavailable}] -> + ets:insert(unavailable, {503, available}), + "HTTP/1.1 503 Service Unavailable\r\n" ++ + "Retry-After:5\r\n" ++ + "Content-Length:47\r\n\r\n" ++ + "<HTML><BODY>Internal Server Error</BODY></HTML>"; + [{503, available}] -> + DefaultResponse; + [{503, long_unavailable}] -> + "HTTP/1.1 503 Service Unavailable\r\n" ++ + "Retry-After:120\r\n" ++ + "Content-Length:47\r\n\r\n" ++ + "<HTML><BODY>Internal Server Error</BODY></HTML>" + end; + +handle_uri(_,"/redirectloop.html",Port,_,Socket,_) -> + %% Create a potential endless loop! + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/redirectloop.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 300 Multiple Choices\r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; + +handle_uri(_,"/userinfo.html", _,Headers,_, DefaultResponse) -> + Challange = "HTTP/1.1 401 Unauthorized \r\n" ++ + "WWW-Authenticate:Basic" ++"\r\n" ++ + "Content-Length:0\r\n\r\n", + case auth_header(Headers) of + {ok, Value} -> + handle_auth(Value, Challange, DefaultResponse); _ -> - tsp("request_and_verify -> local-server *not* running - skip"), - hard_skip("Local http-server not running") + Challange + end; + +handle_uri(_,"/dummy_headers.html",_,_,Socket,_) -> + %% The client will only care about the Transfer-Encoding + %% header the rest of these headers are left to the + %% user to evaluate. This is not a valid response + %% it only tests that the header handling code works. + Head = "HTTP/1.1 200 ok\r\n" ++ + "Content-Length:32\r\n" ++ + "Pragma:1#no-cache\r\n" ++ + "Via:1.0 fred, 1.1 nowhere.com (Apache/1.1)\r\n" ++ + "Warning:1#pseudonym foobar\r\n" ++ + "Vary:*\r\n" ++ + "Trailer:Other:inets_test\r\n" ++ + "Upgrade:HTTP/2.0\r\n" ++ + "Age:4711\r\n" ++ + "Transfer-Encoding:chunked\r\n" ++ + "Content-Encoding:foo\r\n" ++ + "Content-Language:en\r\n" ++ + "Content-Location:http://www.foobar.se\r\n" ++ + "Content-MD5:104528739076276072743283077410617235478\r\n" + ++ + "Content-Range:Sat, 29 Oct 1994 19:43:31 GMT\r\n" ++ + "Expires:Sat, 29 Oct 1994 19:43:31 GMT\r\n" ++ + "Proxy-Authenticate:#1Basic" ++ + "\r\n\r\n", + send(Socket, Head), + send(Socket, http_chunk:encode("<HTML><BODY>fo")), + send(Socket, http_chunk:encode("obar</BODY></HTML>")), + http_chunk:encode_last(); + +handle_uri(_,"/capital_transfer_encoding.html",_,_,Socket,_) -> + Head = "HTTP/1.1 200 ok\r\n" ++ + "Transfer-Encoding:Chunked\r\n\r\n", + send(Socket, Head), + send(Socket, http_chunk:encode("<HTML><BODY>fo")), + send(Socket, http_chunk:encode("obar</BODY></HTML>")), + http_chunk:encode_last(); + +handle_uri(_,"/cookie.html",_,_,_,_) -> + "HTTP/1.1 200 ok\r\n" ++ + "set-cookie:" ++ "test_cookie=true; path=/;" ++ + "max-age=60000\r\n" ++ + "Content-Length:32\r\n\r\n"++ + "<HTML><BODY>foobar</BODY></HTML>"; + +handle_uri(_,"/missing_crlf.html",_,_,_,_) -> + "HTTP/1.1 200 ok" ++ + "Content-Length:32\r\n" ++ + "<HTML><BODY>foobar</BODY></HTML>"; + +handle_uri(_,"/wrong_statusline.html",_,_,_,_) -> + "ok 200 HTTP/1.1\r\n\r\n" ++ + "Content-Length:32\r\n\r\n" ++ + "<HTML><BODY>foobar</BODY></HTML>"; + +handle_uri(_,"/once_chunked.html",_,_,Socket,_) -> + Head = "HTTP/1.1 200 ok\r\n" ++ + "Transfer-Encoding:Chunked\r\n\r\n", + send(Socket, Head), + send(Socket, http_chunk:encode("<HTML><BODY>fo")), + send(Socket, + http_chunk:encode("obar</BODY></HTML>")), + http_chunk:encode_last(); + +handle_uri(_,"/once.html",_,_,Socket,_) -> + Head = "HTTP/1.1 200 ok\r\n" ++ + "Content-Length:32\r\n\r\n", + send(Socket, Head), + send(Socket, "<HTML><BODY>fo"), + test_server:sleep(1000), + send(Socket, "ob"), + test_server:sleep(1000), + send(Socket, "ar</BODY></HTML>"); + +handle_uri(_,"/invalid_http.html",_,_,_,_) -> + "HTTP/1.1 301\r\nDate:Sun, 09 Dec 2007 13:04:18 GMT\r\n" ++ + "Transfer-Encoding:chunked\r\n\r\n"; + +handle_uri(_,"/missing_reason_phrase.html",_,_,_,_) -> + "HTTP/1.1 200\r\n" ++ + "Content-Length: 32\r\n\r\n" + "<HTML><BODY>foobar</BODY></HTML>"; + +handle_uri(_,"/missing_CR.html",_,_,_,_) -> + "HTTP/1.1 200 ok\n" ++ + "Content-Length:32\r\n\n" ++ + "<HTML><BODY>foobar</BODY></HTML>"; + +handle_uri("HEAD",_,_,_,_,_) -> + "HTTP/1.1 200 ok\r\n" ++ + "Content-Length:0\r\n\r\n"; +handle_uri(_,_,_,_,_,DefaultResponse) -> + DefaultResponse. + +url_start(#sslsocket{}) -> + {ok,Host} = inet:gethostname(), + ?TLS_URL_START ++ Host ++ ":"; +url_start(_) -> + {ok,Host} = inet:gethostname(), + ?URL_START ++ Host ++ ":". + +send(#sslsocket{} = S, Msg) -> + ssl:send(S, Msg); +send(S, Msg) -> + gen_tcp:send(S, Msg). + +close(#sslsocket{} = S) -> + ssl:close(S); +close(S) -> + gen_tcp:close(S). + +sockname(#sslsocket{}= S) -> + ssl:sockname(S); +sockname(S) -> + inet:sockname(S). + +receive_streamed_body(RequestId, Body) -> + receive + {http, {RequestId, stream, BinBodyPart}} -> + receive_streamed_body(RequestId, + <<Body/binary, BinBodyPart/binary>>); + {http, {RequestId, stream_end, _Headers}} -> + Body; + {http, Msg} -> + ct:fail(Msg) end. +receive_streamed_body(RequestId, Body, Pid) -> + httpc:stream_next(Pid), + ct:print("~p:receive_streamed_body -> requested next stream ~n", [?MODULE]), + receive + {http, {RequestId, stream, BinBodyPart}} -> + receive_streamed_body(RequestId, + <<Body/binary, BinBodyPart/binary>>, + Pid); + {http, {RequestId, stream_end, _Headers}} -> + Body; + {http, Msg} -> + ct:fail(Msg) + end. +%% ----------------------------------------------------- +%% A sequence number handler +%% The purpose is to be able to pair requests with responses. +start_sequence_number_server() -> + proc_lib:spawn(fun() -> loop_sequence_number(1) end). -not_implemented_yet() -> - exit(not_implemented_yet). +loop_sequence_number(N) -> + receive + shutdown -> + ok; + {From, get_next} -> + From ! {next_is, N}, + loop_sequence_number(N + 1) + end. -p(F) -> - p(F, []). +get_next_sequence_number(SeqNumServer) -> + SeqNumServer ! {self(), get_next}, + receive {next_is, N} -> N end. -p(F, A) -> - io:format("~p ~w:" ++ F ++ "~n", [self(), ?MODULE | A]). +%% ----------------------------------------------------- +%% Client part +%% Sends requests randomly parallel -tsp(F) -> - inets_test_lib:tsp("[~w]" ++ F, [?MODULE]). -tsp(F, A) -> - inets_test_lib:tsp("[~w]" ++ F, [?MODULE|A]). +run_clients(NumClients, ServerPort, SeqNumServer) -> + {ok,Host} = inet:gethostname(), + set_random_seed(), + lists:map( + fun(Id) -> + Req = lists:flatten(io_lib:format("req~3..0w", [get_next_sequence_number(SeqNumServer)])), + Url = ?URL_START ++ Host ++ ":" ++ integer_to_list(ServerPort) ++ "/" ++ Req, + Pid = proc_lib:spawn( + fun() -> + case httpc:request(Url) of + {ok, {{_,200,_}, _, Resp}} -> + ct:print("[~w] 200 response: " + "~p~n", [Id, Resp]), + case lists:prefix(Req++"->", Resp) of + true -> exit(normal); + false -> exit({bad_resp,Req,Resp}) + end; + {ok, {{_,EC,Reason},_,Resp}} -> + ct:print("[~w] ~w response: " + "~s~n~s~n", + [Id, EC, Reason, Resp]), + exit({bad_resp,Req,Resp}); + Crap -> + ct:print("[~w] bad response: ~p", + [Id, Crap]), + exit({bad_resp, Req, Crap}) + end + end), + MRef = erlang:monitor(process, Pid), + timer:sleep(10 + random:uniform(1334)), + {Id, Pid, MRef} + end, + lists:seq(1, NumClients)). -tsf(Reason) -> - test_server:fail(Reason). +wait4clients([], _Timeout) -> + ok; +wait4clients(Clients, Timeout) when Timeout > 0 -> + Time = now_ms(), + receive + {'DOWN', _MRef, process, Pid, normal} -> + {value, {Id, _, _}} = lists:keysearch(Pid, 2, Clients), + NewClients = lists:keydelete(Id, 1, Clients), + wait4clients(NewClients, Timeout - (now_ms() - Time)); + {'DOWN', _MRef, process, Pid, Reason} -> + {value, {Id, _, _}} = lists:keysearch(Pid, 2, Clients), + ct:fail({bad_client_termination, Id, Reason}) + after Timeout -> + ct:fail({client_timeout, Clients}) + end; +wait4clients(Clients, _) -> + ct:fail({client_timeout, Clients}). -dummy_ssl_server_hang(Caller, IpV, SslOpt) -> - Pid = spawn(httpc_SUITE, dummy_ssl_server_hang_init, [Caller, IpV, SslOpt]), +%% ----------------------------------------------------- +%% Webserver part: +%% Implements a web server that sends responses one character +%% at a time, with random delays between the characters. + +start_slow_server(SeqNumServer) -> + proc_lib:start( + erlang, apply, [fun() -> init_slow_server(SeqNumServer) end, []]). + +init_slow_server(SeqNumServer) -> + Inet = inet_version(), + {ok, LSock} = gen_tcp:listen(0, [binary, Inet, {packet,0}, {active,true}, + {backlog, 100}]), + {ok, {_IP, Port}} = inet:sockname(LSock), + proc_lib:init_ack({ok, self(), Port}), + loop_slow_server(LSock, SeqNumServer). + +loop_slow_server(LSock, SeqNumServer) -> + Master = self(), + Acceptor = proc_lib:spawn( + fun() -> client_handler(Master, LSock, SeqNumServer) end), receive - {port, Port} -> - {Pid, Port} + {accepted, Acceptor} -> + loop_slow_server(LSock, SeqNumServer); + shutdown -> + gen_tcp:close(LSock), + exit(Acceptor, kill) end. -dummy_ssl_server_hang_init(Caller, IpV, SslOpt) -> - {ok, ListenSocket} = - case IpV of - ipv4 -> - ssl:listen(0, [binary, inet, {packet, 0}, - {reuseaddr,true}, - {active, false}] ++ SslOpt); - ipv6 -> - ssl:listen(0, [binary, inet6, {packet, 0}, - {reuseaddr,true}, - {active, false}] ++ SslOpt) - end, - {ok, {_,Port}} = ssl:sockname(ListenSocket), - tsp("dummy_ssl_server_hang_init -> Port: ~p", [Port]), - Caller ! {port, Port}, - {ok, AcceptSocket} = ssl:transport_accept(ListenSocket), - dummy_ssl_server_hang_loop(AcceptSocket). -dummy_ssl_server_hang_loop(_) -> - %% Do not do ssl:ssl_accept as we - %% want to time out the underlying gen_tcp:connect +%% Handle one client connection +client_handler(Master, LSock, SeqNumServer) -> + {ok, CSock} = gen_tcp:accept(LSock), + Master ! {accepted, self()}, + set_random_seed(), + loop_client(1, CSock, SeqNumServer). + +loop_client(N, CSock, SeqNumServer) -> + %% Await request, don't bother parsing it too much, + %% assuming the entire request arrives in one packet. receive - stop -> - ok + {tcp, CSock, Req} -> + ReqNum = parse_req_num(Req), + RespSeqNum = get_next_sequence_number(SeqNumServer), + Response = lists:flatten(io_lib:format("~s->resp~3..0w/~2..0w", [ReqNum, RespSeqNum, N])), + Txt = lists:flatten(io_lib:format("Slow server (~p) got ~p, answering with ~p", + [self(), Req, Response])), + ct:print("~s...~n", [Txt]), + slowly_send_response(CSock, Response), + case parse_connection_type(Req) of + keep_alive -> + ct:print("~s...done~n", [Txt]), + loop_client(N+1, CSock, SeqNumServer); + close -> + ct:print("~s...done (closing)~n", [Txt]), + gen_tcp:close(CSock) + end end. -hard_skip(Reason) -> - throw(skip(Reason)). +slowly_send_response(CSock, Answer) -> + Response = lists:flatten(io_lib:format("HTTP/1.1 200 OK\r\nContent-Length: ~w\r\n\r\n~s", + [length(Answer), Answer])), + lists:foreach( + fun(Char) -> + timer:sleep(random:uniform(500)), + gen_tcp:send(CSock, <<Char>>) + end, + Response). -skip(Reason) -> - {skip, Reason}. +parse_req_num(Request) -> + Opts = [caseless,{capture,all_but_first,list}], + {match, [ReqNum]} = re:run(Request, "GET /(.*) HTTP", Opts), + ReqNum. + +parse_connection_type(Request) -> + Opts = [caseless,{capture,all_but_first,list}], + {match,[CType]} = re:run(Request, "connection: *(keep-alive|close)", Opts), + case string:to_lower(CType) of + "close" -> close; + "keep-alive" -> keep_alive + end. + +%% Time in milli seconds +now_ms() -> + {A,B,C} = erlang:now(), + A*1000000000+B*1000+(C div 1000). + +set_random_seed() -> + {_, _, Micros} = now(), + A = erlang:phash2([make_ref(), self(), Micros]), + random:seed(A, A, A). + + +otp_8739(doc) -> + ["OTP-8739"]; +otp_8739(suite) -> + []; +otp_8739(Config) when is_list(Config) -> + {_DummyServerPid, Port} = otp_8739_dummy_server(), + {ok,Host} = inet:gethostname(), + URL = ?URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ "/dummy.html", + Method = get, + Request = {URL, []}, + HttpOptions = [{connect_timeout, 500}, {timeout, 1}], + Options = [{sync, true}], + case httpc:request(Method, Request, HttpOptions, Options) of + {error, timeout} -> + %% And now we check the size of the handler db + Info = httpc:info(), + ct:print("Info: ~p", [Info]), + {value, {handlers, Handlers}} = + lists:keysearch(handlers, 1, Info), + case Handlers of + [] -> + ok; + _ -> + ct:fail({unexpected_handlers, Handlers}) + end; + Unexpected -> + ct:fail({unexpected, Unexpected}) + end. + +otp_8739_dummy_server() -> + Parent = self(), + Pid = spawn_link(fun() -> otp_8739_dummy_server_init(Parent) end), + receive + {port, Port} -> + {Pid, Port} + end. + +otp_8739_dummy_server_init(Parent) -> + Inet = inet_version(), + {ok, ListenSocket} = + gen_tcp:listen(0, [binary, Inet, {packet, 0}, + {reuseaddr,true}, + {active, false}]), + {ok, Port} = inet:port(ListenSocket), + Parent ! {port, Port}, + otp_8739_dummy_server_main(Parent, ListenSocket). + +otp_8739_dummy_server_main(_Parent, ListenSocket) -> + case gen_tcp:accept(ListenSocket) of + {ok, Sock} -> + %% Ignore the request, and simply wait for the socket to close + receive + {tcp_closed, Sock} -> + (catch gen_tcp:close(ListenSocket)), + exit(normal); + {tcp_error, Sock, Reason} -> + ct:fail("socket error: ~p", [Reason]), + (catch gen_tcp:close(ListenSocket)), + exit(normal) + after 10000 -> + %% Just in case + (catch gen_tcp:close(Sock)), + (catch gen_tcp:close(ListenSocket)), + exit(timeout) + end; + Error -> + exit(Error) + end. diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 592469a12f..1efa78a63e 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2012. All Rights Reserved. +%% Copyright Ericsson AB 2005-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -67,97 +67,36 @@ ]). -export([ - pssl_mod_alias/1, essl_mod_alias/1, - - pssl_mod_actions/1, essl_mod_actions/1, - - pssl_mod_security/1, essl_mod_security/1, - - pssl_mod_auth/1, essl_mod_auth/1, - - pssl_mod_auth_api/1, essl_mod_auth_api/1, - - pssl_mod_auth_mnesia_api/1, essl_mod_auth_mnesia_api/1, - - pssl_mod_htaccess/1, essl_mod_htaccess/1, - - pssl_mod_cgi/1, essl_mod_cgi/1, - - pssl_mod_esi/1, essl_mod_esi/1, - - pssl_mod_get/1, essl_mod_get/1, - - pssl_mod_head/1, essl_mod_head/1, - - pssl_mod_all/1, essl_mod_all/1, - - pssl_load_light/1, essl_load_light/1, - - pssl_load_medium/1, essl_load_medium/1, - - pssl_load_heavy/1, essl_load_heavy/1, - - pssl_dos_hostname/1, essl_dos_hostname/1, - - pssl_time_test/1, essl_time_test/1, - - pssl_restart_no_block/1, essl_restart_no_block/1, - - pssl_restart_disturbing_block/1, essl_restart_disturbing_block/1, - - pssl_restart_non_disturbing_block/1, essl_restart_non_disturbing_block/1, - - pssl_block_disturbing_idle/1, essl_block_disturbing_idle/1, - - pssl_block_non_disturbing_idle/1, essl_block_non_disturbing_idle/1, - - pssl_block_503/1, essl_block_503/1, - - pssl_block_disturbing_active/1, essl_block_disturbing_active/1, - - pssl_block_non_disturbing_active/1, essl_block_non_disturbing_active/1, - - pssl_block_disturbing_active_timeout_not_released/1, essl_block_disturbing_active_timeout_not_released/1, - - pssl_block_disturbing_active_timeout_released/1, essl_block_disturbing_active_timeout_released/1, - - pssl_block_non_disturbing_active_timeout_not_released/1, essl_block_non_disturbing_active_timeout_not_released/1, - - pssl_block_non_disturbing_active_timeout_released/1, essl_block_non_disturbing_active_timeout_released/1, - - pssl_block_disturbing_blocker_dies/1, essl_block_disturbing_blocker_dies/1, - - pssl_block_non_disturbing_blocker_dies/1, essl_block_non_disturbing_blocker_dies/1 ]). @@ -242,26 +181,7 @@ groups() -> ip_block_non_disturbing_active_timeout_released, ip_block_disturbing_blocker_dies, ip_block_non_disturbing_blocker_dies]}, - {ssl, [], [{group, pssl}, {group, essl}]}, - {pssl, [], - [pssl_mod_alias, pssl_mod_actions, pssl_mod_security, - pssl_mod_auth, pssl_mod_auth_api, - pssl_mod_auth_mnesia_api, pssl_mod_htaccess, - pssl_mod_cgi, pssl_mod_esi, pssl_mod_get, pssl_mod_head, - pssl_mod_all, pssl_load_light, pssl_load_medium, - pssl_load_heavy, pssl_dos_hostname, pssl_time_test, - pssl_restart_no_block, pssl_restart_disturbing_block, - pssl_restart_non_disturbing_block, - pssl_block_disturbing_idle, - pssl_block_non_disturbing_idle, pssl_block_503, - pssl_block_disturbing_active, - pssl_block_non_disturbing_active, - pssl_block_disturbing_active_timeout_not_released, - pssl_block_disturbing_active_timeout_released, - pssl_block_non_disturbing_active_timeout_not_released, - pssl_block_non_disturbing_active_timeout_released, - pssl_block_disturbing_blocker_dies, - pssl_block_non_disturbing_blocker_dies]}, + {ssl, [], [{group, essl}]}, {essl, [], [essl_mod_alias, essl_mod_actions, essl_mod_security, essl_mod_auth, essl_mod_auth_api, @@ -375,8 +295,8 @@ init_per_testcase(Case, Config) -> init_per_testcase2(Case, Config) -> - tsp("init_per_testcase2 -> entry with" - "~n Config: ~p", [Config]), + %% tsp("init_per_testcase2 -> entry with" + %% "~n Config: ~p", [Config]), IpNormal = integer_to_list(?IP_PORT) ++ ".conf", IpHtaccess = integer_to_list(?IP_PORT) ++ "htaccess.conf", @@ -386,33 +306,33 @@ init_per_testcase2(Case, Config) -> DataDir = ?config(data_dir, Config), SuiteTopDir = ?config(suite_top_dir, Config), - tsp("init_per_testcase2 -> " - "~n SuiteDir: ~p" - "~n DataDir: ~p", [SuiteTopDir, DataDir]), + %% tsp("init_per_testcase2 -> " + %% "~n SuiteDir: ~p" + %% "~n DataDir: ~p", [SuiteTopDir, DataDir]), TcTopDir = filename:join(SuiteTopDir, Case), ?line ok = file:make_dir(TcTopDir), - tsp("init_per_testcase2 -> " - "~n TcTopDir: ~p", [TcTopDir]), + %% tsp("init_per_testcase2 -> " + %% "~n TcTopDir: ~p", [TcTopDir]), DataSrc = filename:join([DataDir, "server_root"]), ServerRoot = filename:join([TcTopDir, "server_root"]), - tsp("init_per_testcase2 -> " - "~n DataSrc: ~p" - "~n ServerRoot: ~p", [DataSrc, ServerRoot]), + %% tsp("init_per_testcase2 -> " + %% "~n DataSrc: ~p" + %% "~n ServerRoot: ~p", [DataSrc, ServerRoot]), ok = file:make_dir(ServerRoot), ok = file:make_dir(filename:join([TcTopDir, "logs"])), NewConfig = [{tc_top_dir, TcTopDir}, {server_root, ServerRoot} | Config], - tsp("init_per_testcase2 -> copy DataSrc to ServerRoot"), + %% tsp("init_per_testcase2 -> copy DataSrc to ServerRoot"), inets_test_lib:copy_dirs(DataSrc, ServerRoot), - tsp("init_per_testcase2 -> fix cgi"), + %% tsp("init_per_testcase2 -> fix cgi"), EnvCGI = filename:join([ServerRoot, "cgi-bin", "printenv.sh"]), {ok, FileInfo} = file:read_file_info(EnvCGI), ok = file:write_file_info(EnvCGI, @@ -432,14 +352,14 @@ init_per_testcase2(Case, Config) -> FileInfo1#file_info{mode = 8#00755}), %% To be used by IP test cases - tsp("init_per_testcase2 -> ip testcase setups"), + %% tsp("init_per_testcase2 -> ip testcase setups"), create_config([{port, ?IP_PORT}, {sock_type, ip_comm} | NewConfig], normal_access, IpNormal), create_config([{port, ?IP_PORT}, {sock_type, ip_comm} | NewConfig], mod_htaccess, IpHtaccess), %% To be used by SSL test cases - tsp("init_per_testcase2 -> ssl testcase setups"), + %% tsp("init_per_testcase2 -> ssl testcase setups"), SocketType = case atom_to_list(Case) of [X, $s, $s, $l | _] -> @@ -504,8 +424,8 @@ init_per_testcase2(Case, Config) -> NewConfig end, - tsp("init_per_testcase2 -> done when" - "~n NewConfig2: ~p", [NewConfig2]), + %% tsp("init_per_testcase2 -> done when" + %% "~n NewConfig2: ~p", [NewConfig2]), NewConfig2. @@ -537,6 +457,7 @@ init_per_testcase3(Case, Config) -> Dog = test_server:timetrap(inets_test_lib:minutes(10)), NewConfig = lists:keydelete(watchdog, 1, Config), TcTopDir = ?config(tc_top_dir, Config), + CaseRest = case atom_to_list(Case) of "ip_mod_htaccess" -> @@ -1069,13 +990,6 @@ ip_restart_non_disturbing_block(Config) when is_list(Config) -> %%------------------------------------------------------------------------- -pssl_mod_alias(doc) -> - ["Module test: mod_alias - old SSL config"]; -pssl_mod_alias(suite) -> - []; -pssl_mod_alias(Config) when is_list(Config) -> - ssl_mod_alias(ssl, Config). - essl_mod_alias(doc) -> ["Module test: mod_alias - using new of configure new SSL"]; essl_mod_alias(suite) -> @@ -1092,13 +1006,6 @@ ssl_mod_alias(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_actions(doc) -> - ["Module test: mod_actions - old SSL config"]; -pssl_mod_actions(suite) -> - []; -pssl_mod_actions(Config) when is_list(Config) -> - ssl_mod_actions(ssl, Config). - essl_mod_actions(doc) -> ["Module test: mod_actions - using new of configure new SSL"]; essl_mod_actions(suite) -> @@ -1117,13 +1024,6 @@ ssl_mod_actions(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_security(doc) -> - ["Module test: mod_security - old SSL config"]; -pssl_mod_security(suite) -> - []; -pssl_mod_security(Config) when is_list(Config) -> - ssl_mod_security(ssl, Config). - essl_mod_security(doc) -> ["Module test: mod_security - using new of configure new SSL"]; essl_mod_security(suite) -> @@ -1143,13 +1043,6 @@ ssl_mod_security(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_auth(doc) -> - ["Module test: mod_auth - old SSL config"]; -pssl_mod_auth(suite) -> - []; -pssl_mod_auth(Config) when is_list(Config) -> - ssl_mod_auth(ssl, Config). - essl_mod_auth(doc) -> ["Module test: mod_auth - using new of configure new SSL"]; essl_mod_auth(suite) -> @@ -1167,12 +1060,6 @@ ssl_mod_auth(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_auth_api(doc) -> - ["Module test: mod_auth - old SSL config"]; -pssl_mod_auth_api(suite) -> - []; -pssl_mod_auth_api(Config) when is_list(Config) -> - ssl_mod_auth_api(ssl, Config). essl_mod_auth_api(doc) -> ["Module test: mod_auth - using new of configure new SSL"]; @@ -1193,12 +1080,6 @@ ssl_mod_auth_api(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_auth_mnesia_api(doc) -> - ["Module test: mod_auth_mnesia_api - old SSL config"]; -pssl_mod_auth_mnesia_api(suite) -> - []; -pssl_mod_auth_mnesia_api(Config) when is_list(Config) -> - ssl_mod_auth_mnesia_api(ssl, Config). essl_mod_auth_mnesia_api(doc) -> ["Module test: mod_auth_mnesia_api - using new of configure new SSL"]; @@ -1217,13 +1098,6 @@ ssl_mod_auth_mnesia_api(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_htaccess(doc) -> - ["Module test: mod_htaccess - old SSL config"]; -pssl_mod_htaccess(suite) -> - []; -pssl_mod_htaccess(Config) when is_list(Config) -> - ssl_mod_htaccess(ssl, Config). - essl_mod_htaccess(doc) -> ["Module test: mod_htaccess - using new of configure new SSL"]; essl_mod_htaccess(suite) -> @@ -1241,13 +1115,6 @@ ssl_mod_htaccess(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_cgi(doc) -> - ["Module test: mod_cgi - old SSL config"]; -pssl_mod_cgi(suite) -> - []; -pssl_mod_cgi(Config) when is_list(Config) -> - ssl_mod_cgi(ssl, Config). - essl_mod_cgi(doc) -> ["Module test: mod_cgi - using new of configure new SSL"]; essl_mod_cgi(suite) -> @@ -1265,13 +1132,6 @@ ssl_mod_cgi(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_esi(doc) -> - ["Module test: mod_esi - old SSL config"]; -pssl_mod_esi(suite) -> - []; -pssl_mod_esi(Config) when is_list(Config) -> - ssl_mod_esi(ssl, Config). - essl_mod_esi(doc) -> ["Module test: mod_esi - using new of configure new SSL"]; essl_mod_esi(suite) -> @@ -1289,13 +1149,6 @@ ssl_mod_esi(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_get(doc) -> - ["Module test: mod_get - old SSL config"]; -pssl_mod_get(suite) -> - []; -pssl_mod_get(Config) when is_list(Config) -> - ssl_mod_get(ssl, Config). - essl_mod_get(doc) -> ["Module test: mod_get - using new of configure new SSL"]; essl_mod_get(suite) -> @@ -1313,13 +1166,6 @@ ssl_mod_get(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_head(doc) -> - ["Module test: mod_head - old SSL config"]; -pssl_mod_head(suite) -> - []; -pssl_mod_head(Config) when is_list(Config) -> - ssl_mod_head(ssl, Config). - essl_mod_head(doc) -> ["Module test: mod_head - using new of configure new SSL"]; essl_mod_head(suite) -> @@ -1337,13 +1183,6 @@ ssl_mod_head(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_mod_all(doc) -> - ["All modules test - old SSL config"]; -pssl_mod_all(suite) -> - []; -pssl_mod_all(Config) when is_list(Config) -> - ssl_mod_all(ssl, Config). - essl_mod_all(doc) -> ["All modules test - using new of configure new SSL"]; essl_mod_all(suite) -> @@ -1361,13 +1200,6 @@ ssl_mod_all(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_load_light(doc) -> - ["Test light load - old SSL config"]; -pssl_load_light(suite) -> - []; -pssl_load_light(Config) when is_list(Config) -> - ssl_load_light(ssl, Config). - essl_load_light(doc) -> ["Test light load - using new of configure new SSL"]; essl_load_light(suite) -> @@ -1386,13 +1218,6 @@ ssl_load_light(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_load_medium(doc) -> - ["Test medium load - old SSL config"]; -pssl_load_medium(suite) -> - []; -pssl_load_medium(Config) when is_list(Config) -> - ssl_load_medium(ssl, Config). - essl_load_medium(doc) -> ["Test medium load - using new of configure new SSL"]; essl_load_medium(suite) -> @@ -1417,13 +1242,6 @@ ssl_load_medium(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_load_heavy(doc) -> - ["Test heavy load - old SSL config"]; -pssl_load_heavy(suite) -> - []; -pssl_load_heavy(Config) when is_list(Config) -> - ssl_load_heavy(ssl, Config). - essl_load_heavy(doc) -> ["Test heavy load - using new of configure new SSL"]; essl_load_heavy(suite) -> @@ -1448,12 +1266,6 @@ ssl_load_heavy(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_dos_hostname(doc) -> - ["Denial Of Service (DOS) attack test case - old SSL config"]; -pssl_dos_hostname(suite) -> - []; -pssl_dos_hostname(Config) when is_list(Config) -> - ssl_dos_hostname(ssl, Config). essl_dos_hostname(doc) -> ["Denial Of Service (DOS) attack test case - using new of configure new SSL"]; @@ -1473,12 +1285,6 @@ ssl_dos_hostname(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_time_test(doc) -> - ["old SSL config"]; -pssl_time_test(suite) -> - []; -pssl_time_test(Config) when is_list(Config) -> - ssl_time_test(ssl, Config). essl_time_test(doc) -> ["using new of configure new SSL"]; @@ -1511,13 +1317,6 @@ ssl_time_test(Tag, Config) when is_list(Config) -> %%------------------------------------------------------------------------- -pssl_block_503(doc) -> - ["Check that you will receive status code 503 when the server" - " is blocked and 200 when its not blocked - old SSL config."]; -pssl_block_503(suite) -> - []; -pssl_block_503(Config) when is_list(Config) -> - ssl_block_503(ssl, Config). essl_block_503(doc) -> ["Check that you will receive status code 503 when the server" @@ -1537,15 +1336,6 @@ ssl_block_503(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_disturbing_idle(doc) -> - ["Check that you can block/unblock an idle server. The strategy " - "distribing does not really make a difference in this case." - "Old SSL config"]; -pssl_block_disturbing_idle(suite) -> - []; -pssl_block_disturbing_idle(Config) when is_list(Config) -> - ssl_block_disturbing_idle(ssl, Config). - essl_block_disturbing_idle(doc) -> ["Check that you can block/unblock an idle server. The strategy " "distribing does not really make a difference in this case." @@ -1565,15 +1355,6 @@ ssl_block_disturbing_idle(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_non_disturbing_idle(doc) -> - ["Check that you can block/unblock an idle server. The strategy " - "non distribing does not really make a difference in this case." - "Old SSL config"]; -pssl_block_non_disturbing_idle(suite) -> - []; -pssl_block_non_disturbing_idle(Config) when is_list(Config) -> - ssl_block_non_disturbing_idle(ssl, Config). - essl_block_non_disturbing_idle(doc) -> ["Check that you can block/unblock an idle server. The strategy " "non distribing does not really make a difference in this case." @@ -1593,15 +1374,6 @@ ssl_block_non_disturbing_idle(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_disturbing_active(doc) -> - ["Check that you can block/unblock an active server. The strategy " - "distribing means ongoing requests should be terminated." - "Old SSL config"]; -pssl_block_disturbing_active(suite) -> - []; -pssl_block_disturbing_active(Config) when is_list(Config) -> - ssl_block_disturbing_active(ssl, Config). - essl_block_disturbing_active(doc) -> ["Check that you can block/unblock an active server. The strategy " "distribing means ongoing requests should be terminated." @@ -1621,15 +1393,6 @@ ssl_block_disturbing_active(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_non_disturbing_active(doc) -> - ["Check that you can block/unblock an idle server. The strategy " - "non distribing means the ongoing requests should be compleated." - "Old SSL config"]; -pssl_block_non_disturbing_active(suite) -> - []; -pssl_block_non_disturbing_active(Config) when is_list(Config) -> - ssl_block_non_disturbing_active(ssl, Config). - essl_block_non_disturbing_active(doc) -> ["Check that you can block/unblock an idle server. The strategy " "non distribing means the ongoing requests should be compleated." @@ -1649,17 +1412,6 @@ ssl_block_non_disturbing_active(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_disturbing_active_timeout_not_released(doc) -> - ["Check that you can block an active server. The strategy " - "distribing means ongoing requests should be compleated" - "if the timeout does not occur." - "Old SSL config"]; -pssl_block_disturbing_active_timeout_not_released(suite) -> - []; -pssl_block_disturbing_active_timeout_not_released(Config) - when is_list(Config) -> - ssl_block_disturbing_active_timeout_not_released(ssl, Config). - essl_block_disturbing_active_timeout_not_released(doc) -> ["Check that you can block an active server. The strategy " "distribing means ongoing requests should be compleated" @@ -1682,17 +1434,6 @@ ssl_block_disturbing_active_timeout_not_released(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_disturbing_active_timeout_released(doc) -> - ["Check that you can block an active server. The strategy " - "distribing means ongoing requests should be terminated when" - "the timeout occurs." - "Old SSL config"]; -pssl_block_disturbing_active_timeout_released(suite) -> - []; -pssl_block_disturbing_active_timeout_released(Config) - when is_list(Config) -> - ssl_block_disturbing_active_timeout_released(ssl, Config). - essl_block_disturbing_active_timeout_released(doc) -> ["Check that you can block an active server. The strategy " "distribing means ongoing requests should be terminated when" @@ -1717,16 +1458,6 @@ ssl_block_disturbing_active_timeout_released(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_non_disturbing_active_timeout_not_released(doc) -> - ["Check that you can block an active server. The strategy " - "non non distribing means ongoing requests should be completed." - "Old SSL config"]; -pssl_block_non_disturbing_active_timeout_not_released(suite) -> - []; -pssl_block_non_disturbing_active_timeout_not_released(Config) - when is_list(Config) -> - ssl_block_non_disturbing_active_timeout_not_released(ssl, Config). - essl_block_non_disturbing_active_timeout_not_released(doc) -> ["Check that you can block an active server. The strategy " "non non distribing means ongoing requests should be completed." @@ -1750,16 +1481,6 @@ ssl_block_non_disturbing_active_timeout_not_released(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_non_disturbing_active_timeout_released(doc) -> - ["Check that you can block an active server. The strategy " - "non distribing means ongoing requests should be completed. " - "When the timeout occurs the block operation sohould be canceled." - "Old SSL config"]; -pssl_block_non_disturbing_active_timeout_released(suite) -> - []; -pssl_block_non_disturbing_active_timeout_released(Config) - when is_list(Config) -> - ssl_block_non_disturbing_active_timeout_released(ssl, Config). essl_block_non_disturbing_active_timeout_released(doc) -> ["Check that you can block an active server. The strategy " @@ -1787,12 +1508,6 @@ ssl_block_non_disturbing_active_timeout_released(Tag, Config) %%------------------------------------------------------------------------- -pssl_block_disturbing_blocker_dies(doc) -> - ["old SSL config"]; -pssl_block_disturbing_blocker_dies(suite) -> - []; -pssl_block_disturbing_blocker_dies(Config) when is_list(Config) -> - ssl_block_disturbing_blocker_dies(ssl, Config). essl_block_disturbing_blocker_dies(doc) -> ["using new of configure new SSL"]; @@ -1811,13 +1526,6 @@ ssl_block_disturbing_blocker_dies(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_block_non_disturbing_blocker_dies(doc) -> - ["old SSL config"]; -pssl_block_non_disturbing_blocker_dies(suite) -> - []; -pssl_block_non_disturbing_blocker_dies(Config) when is_list(Config) -> - ssl_block_non_disturbing_blocker_dies(ssl, Config). - essl_block_non_disturbing_blocker_dies(doc) -> ["using new of configure new SSL"]; essl_block_non_disturbing_blocker_dies(suite) -> @@ -1835,12 +1543,6 @@ ssl_block_non_disturbing_blocker_dies(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_restart_no_block(doc) -> - ["old SSL config"]; -pssl_restart_no_block(suite) -> - []; -pssl_restart_no_block(Config) when is_list(Config) -> - ssl_restart_no_block(ssl, Config). essl_restart_no_block(doc) -> ["using new of configure new SSL"]; @@ -1859,12 +1561,6 @@ ssl_restart_no_block(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_restart_disturbing_block(doc) -> - ["old SSL config"]; -pssl_restart_disturbing_block(suite) -> - []; -pssl_restart_disturbing_block(Config) when is_list(Config) -> - ssl_restart_disturbing_block(ssl, Config). essl_restart_disturbing_block(doc) -> ["using new of configure new SSL"]; @@ -1916,12 +1612,6 @@ ssl_restart_disturbing_block(Tag, Config) -> %%------------------------------------------------------------------------- -pssl_restart_non_disturbing_block(doc) -> - ["old SSL config"]; -pssl_restart_non_disturbing_block(suite) -> - []; -pssl_restart_non_disturbing_block(Config) when is_list(Config) -> - ssl_restart_non_disturbing_block(ssl, Config). essl_restart_non_disturbing_block(doc) -> ["using new of configure new SSL"]; @@ -2314,7 +2004,7 @@ create_config(Config, Access, FileName) -> "~n Type: ~p" "~n Port: ~p" "~n Host: ~p" - "~n", [ServerRoot, TcTopDir, Port, Type, Host]), + "~n", [ServerRoot, TcTopDir, Type, Port, Host]), SSL = if @@ -2752,3 +2442,4 @@ tsp(F, A) -> tsf(Reason) -> inets_test_lib:tsf(Reason). + diff --git a/lib/inets/test/httpd_mod.erl b/lib/inets/test/httpd_mod.erl index 387263ce58..df4ed6b179 100644 --- a/lib/inets/test/httpd_mod.erl +++ b/lib/inets/test/httpd_mod.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2012. All Rights Reserved. +%% Copyright Ericsson AB 2005-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -40,7 +40,6 @@ %%------------------------------------------------------------------------- alias(Type, Port, Host, Node) -> %% This is very crude, but... - tsp("alias -> Has IPv6 support: ~p", [inets_test_lib:has_ipv6_support()]), Opts = [], ok = httpd_test_lib:verify_request(Type, Host, Port, Opts, Node, "GET /pics/icon.sheet.gif " @@ -85,16 +84,7 @@ actions(Type, Port, Host, Node) -> %%------------------------------------------------------------------------- security(ServerRoot, Type, Port, Host, Node) -> - tsp("security -> " - "entry with" - "~n ServerRoot: ~p" - "~n Type: ~p" - "~n Port: ~p" - "~n Host: ~p" - "~n Node: ~p", [ServerRoot, Type, Port, Host, Node]), - - tsp("security -> " - "register - receive security events"), + global:register_name(mod_security_test, self()), % Receive events tsp("security -> " @@ -333,13 +323,7 @@ security(ServerRoot, Type, Port, Host, Node) -> %%------------------------------------------------------------------------- auth(Type, Port, Host, Node) -> - tsp("auth -> " - "entry with" - "~n Type: ~p" - "~n Port: ~p" - "~n Host: ~p" - "~n Node: ~p", [Type, Port, Host, Node]), - + %% Authentication required! ok = httpd_test_lib:verify_request(Type,Host,Port,Node, "GET /open/ HTTP/1.0\r\n\r\n", @@ -750,11 +734,6 @@ htaccess(Type, Port, Host, Node) -> {header, "WWW-Authenticate"}]). %%-------------------------------------------------------------------- cgi(Type, Port, Host, Node) -> -%% tsp("cgi -> entry with" -%% "~n Type: ~p" -%% "~n Port: ~p" -%% "~n Host: ~p" -%% "~n Node: ~p", []), {Script, Script2, Script3} = case test_server:os_type() of {win32, _} -> diff --git a/lib/inets/test/httpd_test_lib.erl b/lib/inets/test/httpd_test_lib.erl index 4b33350cf2..13584c50f6 100644 --- a/lib/inets/test/httpd_test_lib.erl +++ b/lib/inets/test/httpd_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2012. All Rights Reserved. +%% Copyright Ericsson AB 2001-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -91,7 +91,7 @@ verify_request(SocketType, Host, Port, Node, RequestStr, Options, TimeOut) when (is_integer(TimeOut) orelse (TimeOut =:= infinity)) -> verify_request(SocketType, Host, Port, [], Node, RequestStr, Options, TimeOut). -verify_request(SocketType, Host, Port, TranspOpts, Node, RequestStr, Options, TimeOut) -> +verify_request(SocketType, Host, Port, TranspOpts0, Node, RequestStr, Options, TimeOut) -> tsp("verify_request -> entry with" "~n SocketType: ~p" "~n Host: ~p" @@ -100,7 +100,17 @@ verify_request(SocketType, Host, Port, TranspOpts, Node, RequestStr, Options, Ti "~n Node: ~p" "~n Options: ~p" "~n TimeOut: ~p", - [SocketType, Host, Port, TranspOpts, Node, Options, TimeOut]), + [SocketType, Host, Port, TranspOpts0, Node, Options, TimeOut]), + + %% For now, until we modernize the httpd tests + TranspOpts = + case lists:member(inet6, TranspOpts0) of + true -> + TranspOpts0; + false -> + [inet | TranspOpts0] + end, + try inets_test_lib:connect_bin(SocketType, Host, Port, TranspOpts) of {ok, Socket} -> tsp("verify_request -> connected - now send message"), @@ -293,8 +303,7 @@ validate(RequestStr, #state{status_line = {Version, StatusCode, _}, list_to_integer(Headers#http_response_h.'content-length'), Body). - -%%-------------------------------------------------------------------- +%-------------------------------------------------------------------- %% Internal functions %%------------------------------------------------------------------ check_version(Version, Options) -> diff --git a/lib/inets/test/inets_appup_test.erl b/lib/inets/test/inets_appup_test.erl index 648e373312..d563b52ae7 100644 --- a/lib/inets/test/inets_appup_test.erl +++ b/lib/inets/test/inets_appup_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2010. All Rights Reserved. +%% Copyright Ericsson AB 2002-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/inets/test/inets_test_lib.erl b/lib/inets/test/inets_test_lib.erl index 0f8671b682..6ccc7b0da1 100644 --- a/lib/inets/test/inets_test_lib.erl +++ b/lib/inets/test/inets_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2012. All Rights Reserved. +%% Copyright Ericsson AB 2001-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,26 +22,8 @@ -include("inets_test_lib.hrl"). -include_lib("inets/src/http_lib/http_internal.hrl"). -%% Various small utility functions --export([start_http_server/1, start_http_server/2]). --export([start_http_server_ssl/1, start_http_server_ssl/2]). --export([hostname/0]). --export([connect_bin/3, connect_bin/4, - connect_byte/3, connect_byte/4, - send/3, close/2]). --export([copy_file/3, copy_files/2, copy_dirs/2, del_dirs/1]). --export([info/4, log/4, debug/4, print/4]). --export([timestamp/0, formated_timestamp/0]). --export([tsp/1, tsp/2, tsf/1, tss/1]). --export([check_body/1]). --export([millis/0, millis_diff/2, hours/1, minutes/1, seconds/1, sleep/1]). --export([oscmd/1, has_ipv6_support/0, has_ipv6_support/1, print_system_info/1]). --export([run_on_os/2, run_on_windows/1]). --export([ensure_started/1]). --export([non_pc_tc_maybe_skip/4, os_based_skip/1, skip/3, fail/3]). --export([flush/0]). --export([start_node/1, stop_node/1]). - +%% Note: This directive should only be used in test suites. +-compile(export_all). %% -- Misc os command and stuff @@ -474,7 +456,7 @@ connect_bin(ssl, Host, Port, Opts0) -> Opts = [binary, {packet,0} | Opts0], connect(ssl, Host, Port, Opts); connect_bin(essl, Host, Port, Opts0) -> - Opts = [{ssl_imp, new}, binary, {packet,0}, {reuseaddr, true} | Opts0], + Opts = [{ssl_imp, new}, binary, {packet,0}| Opts0], connect(ssl, Host, Port, Opts); connect_bin(ip_comm, Host, Port, Opts0) -> Opts = [binary, {packet, 0} | Opts0], @@ -494,74 +476,10 @@ connect_byte(ip_comm, Host, Port, Opts0) -> Opts = [{packet,0} | Opts0], connect(ip_comm, Host, Port, Opts). - -%% This always falls back on IPV4, but tries IPV6 first. -connect(Proto, Host, Port, Opts0) -> - Opts = Opts0 -- [inet, inet6], - connect(Proto, Host, Port, Opts ++ [inet6], inet6). - -connect(ssl, Host, Port, Opts, Type) -> - tsp("connect(ssl) -> entry with" - "~n Host: ~p" - "~n Port: ~p" - "~n Opts: ~p" - "~n Type: ~p", [Host, Port, Opts, Type]), - ssl:start(), - %% We ignore this option for ssl... - %% ...maybe we should really treat this in the same way as ip_comm... - case ssl:connect(Host, Port, Opts) of - {ok, Socket} -> - {ok, Socket}; - {error, Reason} when Type =:= inet6 -> - tsp("connect(ssl) -> failed connecting with inet6: " - "~n Reason: ~p" - "~n trying inet", [Reason]), - connect(ssl, Host, Port, Opts -- [inet6], inet); - {error, Reason} -> - tsp("connect(ssl) -> failed connecting: " - "~n Reason: ~p", [Reason]), - {error, Reason}; - Error -> - Error - end; -connect(ip_comm, Host, Port, Opts, Type) -> - tsp("connect(ip_comm) -> entry with" - "~n Host: ~p" - "~n Port: ~p" - "~n Opts: ~p" - "~n Type: ~p", [Host, Port, Opts, Type]), - - case gen_tcp:connect(Host, Port, Opts, timer:seconds(10)) of - {ok, Socket} -> - tsp("connect success"), - {ok, Socket}; - - {error, Reason} when ((Type =:= inet6) andalso - ((Reason =:= timeout) orelse - (Reason =:= nxdomain) orelse - (Reason =:= eafnosupport) orelse - (Reason =:= econnreset) orelse - (Reason =:= enetunreach) orelse - (Reason =:= econnrefused) orelse - (Reason =:= ehostunreach))) -> - tsp("connect(ip_comm) -> Connect error: " - "~n Reason: ~p" - "~n Type: ~p" - "~n Opts: ~p", [Reason, Type, Opts]), - connect(ip_comm, Host, Port, Opts -- [inet6], inet); - - Error -> - tsp("connect(ip_comm) -> Fatal connect error: " - "~n Error: ~p" - "~nwhen" - "~n Host: ~p" - "~n Port: ~p" - "~n Opts: ~p" - "~n Type: ~p" - "~n", [Error, Host, Port, Opts, Type]), - Error - end. - +connect(ip_comm, Host, Port, Opts) -> + gen_tcp:connect(Host, Port, Opts); +connect(ssl, Host, Port, Opts) -> + ssl:connect(Host, Port, Opts). send(ssl, Socket, Data) -> ssl:send(Socket, Data); @@ -651,3 +569,13 @@ format_timestamp({_N1, _N2, N3} = Now) -> [YYYY,MM,DD,Hour,Min,Sec,round(N3/1000)]), lists:flatten(FormatDate). +start_apps(Apps) -> + lists:foreach(fun(App) -> + application:stop(App), + application:start(App) + end, Apps). +stop_apps(Apps) -> + lists:foreach(fun(App) -> + application:stop(App) + end, Apps). + diff --git a/lib/inets/test/uri_SUITE.erl b/lib/inets/test/uri_SUITE.erl new file mode 100644 index 0000000000..9ba09e1474 --- /dev/null +++ b/lib/inets/test/uri_SUITE.erl @@ -0,0 +1,159 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +%% +%% ct:run("../inets_test", uri_SUITE). +%% + +-module(uri_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include("inets_test_lib.hrl"). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-define(GOOGLE, "www.google.com"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +suite() -> + [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + ipv4, + ipv6, + host, + userinfo, + scheme, + queries, + escaped, + hexed_query + ]. + +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + Config. +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- + +init_per_testcase(_Case, Config) -> + Config. +end_per_testcase(_Case, _Config) -> + ok. + +%%------------------------------------------------------------------------- +%% Test cases starts here. +%%------------------------------------------------------------------------- + +ipv4(Config) when is_list(Config) -> + {ok, {http,[],"127.0.0.1",80,"/foobar.html",[]}} = + http_uri:parse("http://127.0.0.1/foobar.html"). + +ipv6(Config) when is_list(Config) -> + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html"), + {ok, {http,[],"[2010:836B:4179::836B:4179]",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{ipv6_host_with_brackets, true}]), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{ipv6_host_with_brackets, false}]), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{foo, false}]), + {error, + {malformed_url, _, "http://2010:836B:4179::836B:4179/foobar.html"}} = + http_uri:parse("http://2010:836B:4179::836B:4179/foobar.html"). + +host(Config) when is_list(Config) -> + {ok, {http,[],"localhost",8888,"/foobar.html",[]}} = + http_uri:parse("http://localhost:8888/foobar.html"). + +userinfo(Config) when is_list(Config) -> + {ok, {http,"nisse:foobar","localhost",8888,"/foobar.html",[]}} = + http_uri:parse("http://nisse:foobar@localhost:8888/foobar.html"). + +scheme(Config) when is_list(Config) -> + {error, no_scheme} = http_uri:parse("localhost/foobar.html"), + {error, {malformed_url, _, _}} = + http_uri:parse("localhost:8888/foobar.html"). + +queries(Config) when is_list(Config) -> + {ok, {http,[],"localhost",8888,"/foobar.html","?foo=bar&foobar=42"}} = + http_uri:parse("http://localhost:8888/foobar.html?foo=bar&foobar=42"). + +escaped(Config) when is_list(Config) -> + {ok, {http,[],"www.somedomain.com",80,"/%2Eabc",[]}} = + http_uri:parse("http://www.somedomain.com/%2Eabc"), + {ok, {http,[],"www.somedomain.com",80,"/%252Eabc",[]}} = + http_uri:parse("http://www.somedomain.com/%252Eabc"), + {ok, {http,[],"www.somedomain.com",80,"/%25abc",[]}} = + http_uri:parse("http://www.somedomain.com/%25abc"), + {ok, {http,[],"www.somedomain.com",80,"/%25abc", "?foo=bar"}} = + http_uri:parse("http://www.somedomain.com/%25abc?foo=bar"). + +hexed_query(doc) -> + [{doc, "Solves OTP-6191"}]; +hexed_query(Config) when is_list(Config) -> + Google = ?GOOGLE, + GoogleSearch = "http://" ++ Google ++ "/search", + Search1 = "?hl=en&q=a%D1%85%D1%83%D0%B9&btnG=Google+Search", + URI1 = GoogleSearch ++ Search1, + Search2 = "?hl=en&q=%25%25", + URI2 = GoogleSearch ++ Search2, + Search3 = "?hl=en&q=%foo", + URI3 = GoogleSearch ++ Search3, + + Verify1 = + fun({http, [], ?GOOGLE, 80, "/search", _}) -> ok; + (_) -> error + end, + Verify2 = Verify1, + Verify3 = Verify1, + verify_uri(URI1, Verify1), + verify_uri(URI2, Verify2), + verify_uri(URI3, Verify3). + + +%%-------------------------------------------------------------------- +%% Internal Functions ------------------------------------------------ +%%-------------------------------------------------------------------- + + +verify_uri(URI, Verify) -> + case http_uri:parse(URI) of + {ok, ParsedURI} -> + case Verify(ParsedURI) of + ok -> + ok; + error -> + Reason = {unexpected_parse_result, URI, ParsedURI}, + ERROR = {error, Reason}, + throw(ERROR) + end; + {error, _} = ERROR -> + throw(ERROR) + end. diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 394dccab48..f5abaf9764 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 5.9.3 +INETS_VSN = 5.9.4 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/jinterface/doc/src/notes.xml b/lib/jinterface/doc/src/notes.xml index f0444ac879..25d4482b76 100644 --- a/lib/jinterface/doc/src/notes.xml +++ b/lib/jinterface/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2000</year><year>2012</year> + <year>2000</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,42 @@ </header> <p>This document describes the changes made to the Jinterface application.</p> +<section><title>Jinterface 1.5.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in OtpErlangTuple constructor. Thanks to Vlad + Dumitrescu.</p> + <p> + Own Id: OTP-10819</p> + </item> + <item> + <p> + Fixed finding cookie file on windows. Thanks to Vlad + Dumitrescu</p> + <p> + Own Id: OTP-10821</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Don't compress external binary format if this increases + the size. Thanks to Nico Kruber.</p> + <p> + Own Id: OTP-10822</p> + </item> + </list> + </section> + +</section> + <section><title>Jinterface 1.5.7</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangTuple.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangTuple.java index b97af5cfa0..bffce7f14d 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangTuple.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangTuple.java @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/kernel/doc/src/code.xml b/lib/kernel/doc/src/code.xml index dbcf376487..6f04741f85 100644 --- a/lib/kernel/doc/src/code.xml +++ b/lib/kernel/doc/src/code.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2012</year> + <year>1996</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 069b13eacf..66ecba1bf2 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -60,14 +60,6 @@ converted, why the Unicode mode for file names is not default on systems having completely transparent file naming.</p> - <note> - <p>As of R14B01, the most basic file handling modules - (<c>file</c>, <c>prim_file</c>, <c>filelib</c> and - <c>filename</c>) accept raw file names, but the rest of OTP is not - guaranteed to handle them, why Unicode file naming on systems - where it is not default is still considered experimental.</p> - </note> - <p>Raw file names is a new feature in OTP R14B01, which allows the user to supply completely uninterpreted file names to the underlying OS/filesystem. They are supplied as binaries, where it @@ -125,6 +117,14 @@ <desc> <p>If VM is in Unicode filename mode, <c>string()</c> and <c>char()</c> are allowed to be > 255. + </p> + </desc> + </datatype> + <datatype> + <name name="name_all"/> + <desc> + <p>If VM is in Unicode filename mode, <c>string()</c> and <c>char()</c> + are allowed to be > 255. <c><anno>RawFilename</anno></c> is a filename not subject to Unicode translation, meaning that it can contain characters not conforming to @@ -504,7 +504,8 @@ <name name="list_dir" arity="1"/> <fsummary>List files in a directory</fsummary> <desc> - <p>Lists all the files in a directory. Returns + <p>Lists all files in a directory, <b>except</b> files + with "raw" names. Returns <c>{ok, <anno>Filenames</anno>}</c> if successful. Otherwise, it returns <c>{error, <anno>Reason</anno>}</c>. <c><anno>Filenames</anno></c> is a list of @@ -521,6 +522,37 @@ <item> <p>The directory does not exist.</p> </item> + <tag><c>{no_translation, <anno>Filename</anno>}</c></tag> + <item> + <p><c><anno>Filename</anno></c> is a <c>binary()</c> with + characters coded in ISO-latin-1 and the VM was started + with the parameter <c>+fnue</c>.</p> + </item> + </taglist> + </desc> + </func> + <func> + <name name="list_dir_all" arity="1"/> + <fsummary>List all files in a directory</fsummary> + <desc> + <p>Lists all the files in a directory, including files with + "raw" names. + Returns <c>{ok, <anno>Filenames</anno>}</c> if successful. + Otherwise, it returns <c>{error, <anno>Reason</anno>}</c>. + <c><anno>Filenames</anno></c> is a list of + the names of all the files in the directory. The names are + not sorted.</p> + <p>Typical error reasons are:</p> + <taglist> + <tag><c>eacces</c></tag> + <item> + <p>Missing search or write permissions for <c><anno>Dir</anno></c> + or one of its parent directories.</p> + </item> + <tag><c>enoent</c></tag> + <item> + <p>The directory does not exist.</p> + </item> </taglist> </desc> </func> @@ -1409,10 +1441,41 @@ <fsummary>See what a link is pointing to</fsummary> <desc> <p>This function returns <c>{ok, <anno>Filename</anno>}</c> if + <c><anno>Name</anno></c> refers to a symbolic link that is + not a "raw" file name, or <c>{error, <anno>Reason</anno>}</c> + otherwise. + On platforms that do not support symbolic links, the return + value will be <c>{error,enotsup}</c>.</p> + <p>Typical error reasons:</p> + <taglist> + <tag><c>einval</c></tag> + <item> + <p><c><anno>Name</anno></c> does not refer to a symbolic link + or the name of the file that it refers to does not conform + to the expected encoding.</p> + </item> + <tag><c>enoent</c></tag> + <item> + <p>The file does not exist.</p> + </item> + <tag><c>enotsup</c></tag> + <item> + <p>Symbolic links are not supported on this platform.</p> + </item> + </taglist> + </desc> + </func> + <func> + <name name="read_link_all" arity="1"/> + <fsummary>See what a link is pointing to</fsummary> + <desc> + <p>This function returns <c>{ok, <anno>Filename</anno>}</c> if <c><anno>Name</anno></c> refers to a symbolic link or <c>{error, <anno>Reason</anno>}</c> otherwise. On platforms that do not support symbolic links, the return value will be <c>{error,enotsup}</c>.</p> + <p>Note that <c><anno>Filename</anno></c> can be either a list + or a binary.</p> <p>Typical error reasons:</p> <taglist> <tag><c>einval</c></tag> @@ -1580,6 +1643,12 @@ <p><c><anno>Dir</anno></c> had an improper type, such as tuple.</p> </item> + <tag><c>no_translation</c></tag> + <item> + <p><c><anno>Dir</anno></c> is a <c>binary()</c> with + characters coded in ISO-latin-1 and the VM was started + with the parameter <c>+fnue</c>.</p> + </item> </taglist> <warning> <p>In a future release, a bad type for the diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 6968b36eac..c18ae897b4 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,80 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 2.16.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> A bug that could cause a crash with wrong reason has + been corrected in the <c>application_controller</c> + module. </p> + <p> + Own Id: OTP-10754</p> + </item> + <item> + <p> + Fix <c>code:is_module_native/1</c> that sometimes in R16A + returned false for hipe compiled modules containing BIFs + such as <c>lists</c>.</p> + <p> + Own Id: OTP-10870</p> + </item> + <item> + <p> + Respect <c>{exit_on_close,false}</c> option on tcp socket + in non-passive mode when receiving fails (due to an + ill-formed packet for example) by only doing a half close + and still allow sending on the socket. (Thanks to Anthony + Molinaro and Steve Vinoski for reporting the problem)</p> + <p> + Own Id: OTP-10879</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Slightly nicer error message when node start fails due to + duplicate name. Thanks to Magnus Henoch.</p> + <p> + Own Id: OTP-10797</p> + </item> + <item> + <p> Miscellaneous updates due to Unicode support. </p> + <p> + Own Id: OTP-10820</p> + </item> + <item> + <p> + Add a new function code:get_mode() can be used to detect + how the code servers behaves. Thanks to Vlad Dumitrescu</p> + <p> + Own Id: OTP-10823</p> + </item> + <item> + <p> + Fix type of error Reason on gen_tcp:send/2. Thanks to + Sean Cribbs.</p> + <p> + Own Id: OTP-10839</p> + </item> + <item> + <p><c>file:list_dir_all/1</c> and + <c>file:read_link_all/1</c> that can handle raw file + names have been added. See the User Guide for STDLIB for + information about raw file names.</p> + <p> + Own Id: OTP-10852</p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 2.16</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/auth.erl b/lib/kernel/src/auth.erl index 1e12a647d7..7d463103e3 100644 --- a/lib/kernel/src/auth.erl +++ b/lib/kernel/src/auth.erl @@ -391,7 +391,7 @@ create_cookie(Name) -> {error,Reason} -> {error, lists:flatten( - io_lib:format("Failed to create cookie file '~s': ~p", [Name, Reason]))} + io_lib:format("Failed to create cookie file '~ts': ~p", [Name, Reason]))} end. random_cookie(0, _, Result) -> diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 5f52f94270..a4c56b346f 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -68,7 +68,7 @@ %% Types that can be used from other modules -- alphabetically ordered. -export_type([date_time/0, fd/0, file_info/0, filename/0, filename_all/0, - io_device/0, name/0, posix/0]). + io_device/0, name/0, name_all/0, posix/0]). %%% Includes and defines -include("file.hrl"). @@ -97,7 +97,8 @@ | 'read_ahead' | 'compressed' | {'encoding', unicode:encoding()}. -type deep_list() :: [char() | atom() | deep_list()]. --type name() :: string() | atom() | deep_list() | (RawFilename :: binary()). +-type name() :: string() | atom() | deep_list(). +-type name_all() :: string() | atom() | deep_list() | (RawFilename :: binary()). -type posix() :: 'eacces' | 'eagain' | 'ebadf' | 'ebusy' | 'edquot' | 'eexist' | 'efault' | 'efbig' | 'eintr' | 'einval' | 'eio' | 'eisdir' | 'eloop' | 'emfile' | 'emlink' @@ -117,7 +118,7 @@ -export([file_info/1, native_name_encoding/0]). -spec file_info(Filename) -> {ok, FileInfo} | {error, Reason} when - Filename :: name(), + Filename :: name_all(), FileInfo :: file_info(), Reason :: posix() | badarg. @@ -160,7 +161,7 @@ format_error(ErrorId) -> erl_posix_msg:message(ErrorId). -spec pid2name(Pid) -> {ok, Filename} | undefined when - Filename :: filename(), + Filename :: filename_all(), Pid :: pid(). pid2name(Pid) when is_pid(Pid) -> @@ -197,42 +198,42 @@ get_cwd(Drive) -> -spec set_cwd(Dir) -> ok | {error, Reason} when Dir :: name(), - Reason :: posix() | badarg. + Reason :: posix() | badarg | no_translation. set_cwd(Dirname) -> check_and_call(set_cwd, [file_name(Dirname)]). -spec delete(Filename) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Reason :: posix() | badarg. delete(Name) -> check_and_call(delete, [file_name(Name)]). -spec rename(Source, Destination) -> ok | {error, Reason} when - Source :: name(), - Destination :: name(), + Source :: name_all(), + Destination :: name_all(), Reason :: posix() | badarg. rename(From, To) -> check_and_call(rename, [file_name(From), file_name(To)]). -spec make_dir(Dir) -> ok | {error, Reason} when - Dir :: name(), + Dir :: name_all(), Reason :: posix() | badarg. make_dir(Name) -> check_and_call(make_dir, [file_name(Name)]). -spec del_dir(Dir) -> ok | {error, Reason} when - Dir :: name(), + Dir :: name_all(), Reason :: posix() | badarg. del_dir(Name) -> check_and_call(del_dir, [file_name(Name)]). -spec read_file_info(Filename) -> {ok, FileInfo} | {error, Reason} when - Filename :: name(), + Filename :: name_all(), FileInfo :: file_info(), Reason :: posix() | badarg. @@ -240,7 +241,7 @@ read_file_info(Name) -> check_and_call(read_file_info, [file_name(Name)]). -spec read_file_info(Filename, Opts) -> {ok, FileInfo} | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Opts :: [file_info_option()], FileInfo :: file_info(), Reason :: posix() | badarg. @@ -248,13 +249,13 @@ read_file_info(Name) -> read_file_info(Name, Opts) when is_list(Opts) -> check_and_call(read_file_info, [file_name(Name), Opts]). --spec altname(Name :: name()) -> any(). +-spec altname(Name :: name_all()) -> any(). altname(Name) -> check_and_call(altname, [file_name(Name)]). -spec read_link_info(Name) -> {ok, FileInfo} | {error, Reason} when - Name :: name(), + Name :: name_all(), FileInfo :: file_info(), Reason :: posix() | badarg. @@ -262,7 +263,7 @@ read_link_info(Name) -> check_and_call(read_link_info, [file_name(Name)]). -spec read_link_info(Name, Opts) -> {ok, FileInfo} | {error, Reason} when - Name :: name(), + Name :: name_all(), Opts :: [file_info_option()], FileInfo :: file_info(), Reason :: posix() | badarg. @@ -272,7 +273,7 @@ read_link_info(Name, Opts) when is_list(Opts) -> -spec read_link(Name) -> {ok, Filename} | {error, Reason} when - Name :: name(), + Name :: name_all(), Filename :: filename(), Reason :: posix() | badarg. @@ -280,7 +281,7 @@ read_link(Name) -> check_and_call(read_link, [file_name(Name)]). -spec read_link_all(Name) -> {ok, Filename} | {error, Reason} when - Name :: name(), + Name :: name_all(), Filename :: filename_all(), Reason :: posix() | badarg. @@ -288,7 +289,7 @@ read_link_all(Name) -> check_and_call(read_link_all, [file_name(Name)]). -spec write_file_info(Filename, FileInfo) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), FileInfo :: file_info(), Reason :: posix() | badarg. @@ -296,7 +297,7 @@ write_file_info(Name, Info = #file_info{}) -> check_and_call(write_file_info, [file_name(Name), Info]). -spec write_file_info(Filename, FileInfo, Opts) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Opts :: [file_info_option()], FileInfo :: file_info(), Reason :: posix() | badarg. @@ -305,15 +306,17 @@ write_file_info(Name, Info = #file_info{}, Opts) when is_list(Opts) -> check_and_call(write_file_info, [file_name(Name), Info, Opts]). -spec list_dir(Dir) -> {ok, Filenames} | {error, Reason} when - Dir :: name(), + Dir :: name_all(), Filenames :: [filename()], - Reason :: posix() | badarg. + Reason :: posix() + | badarg + | {no_translation, Filename :: unicode:latin1_binary()}. list_dir(Name) -> check_and_call(list_dir, [file_name(Name)]). -spec list_dir_all(Dir) -> {ok, Filenames} | {error, Reason} when - Dir :: name(), + Dir :: name_all(), Filenames :: [filename_all()], Reason :: posix() | badarg. @@ -321,7 +324,7 @@ list_dir_all(Name) -> check_and_call(list_dir_all, [file_name(Name)]). -spec read_file(Filename) -> {ok, Binary} | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Binary :: binary(), Reason :: posix() | badarg | terminated | system_limit. @@ -329,23 +332,23 @@ read_file(Name) -> check_and_call(read_file, [file_name(Name)]). -spec make_link(Existing, New) -> ok | {error, Reason} when - Existing :: name(), - New :: name(), + Existing :: name_all(), + New :: name_all(), Reason :: posix() | badarg. make_link(Old, New) -> check_and_call(make_link, [file_name(Old), file_name(New)]). -spec make_symlink(Existing, New) -> ok | {error, Reason} when - Existing :: name(), - New :: name(), + Existing :: name_all(), + New :: name_all(), Reason :: posix() | badarg. make_symlink(Old, New) -> check_and_call(make_symlink, [file_name(Old), file_name(New)]). -spec write_file(Filename, Bytes) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Bytes :: iodata(), Reason :: posix() | badarg | terminated | system_limit. @@ -357,7 +360,7 @@ write_file(Name, Bin) -> %% Meanwhile, it is implemented here, slightly less efficient. -spec write_file(Filename, Bytes, Modes) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Bytes :: iodata(), Modes :: [mode()], Reason :: posix() | badarg | terminated | system_limit. @@ -416,7 +419,7 @@ raw_write_file_info(Name, #file_info{} = Info) -> -spec open(File, Modes) -> {ok, IoDevice} | {error, Reason} when File :: Filename | iodata(), - Filename :: name(), + Filename :: name_all(), Modes :: [mode() | ram], IoDevice :: io_device(), Reason :: posix() | badarg | system_limit. @@ -523,7 +526,10 @@ allocate(#file_descriptor{module = Module} = Handle, Offset, Length) -> IoDevice :: io_device() | atom(), Number :: non_neg_integer(), Data :: string() | binary(), - Reason :: posix() | badarg | terminated. + Reason :: posix() + | badarg + | terminated + | {no_translation, unicode, latin1}. read(File, Sz) when (is_pid(File) orelse is_atom(File)), is_integer(Sz), Sz >= 0 -> case io:request(File, {get_chars, '', Sz}) of @@ -541,7 +547,10 @@ read(_, _) -> -spec read_line(IoDevice) -> {ok, Data} | eof | {error, Reason} when IoDevice :: io_device() | atom(), Data :: string() | binary(), - Reason :: posix() | badarg | terminated. + Reason :: posix() + | badarg + | terminated + | {no_translation, unicode, latin1}. read_line(File) when (is_pid(File) orelse is_atom(File)) -> case io:request(File, {get_line, ''}) of @@ -709,7 +718,7 @@ truncate(_) -> -spec copy(Source, Destination) -> {ok, BytesCopied} | {error, Reason} when Source :: io_device() | Filename | {Filename, Modes}, Destination :: io_device() | Filename | {Filename, Modes}, - Filename :: name(), + Filename :: name_all(), Modes :: [mode()], BytesCopied :: non_neg_integer(), Reason :: posix() | badarg | terminated. @@ -721,7 +730,7 @@ copy(Source, Dest) -> {ok, BytesCopied} | {error, Reason} when Source :: io_device() | Filename | {Filename, Modes}, Destination :: io_device() | Filename | {Filename, Modes}, - Filename :: name(), + Filename :: name_all(), Modes :: [mode()], ByteCount :: non_neg_integer() | infinity, BytesCopied :: non_neg_integer(), @@ -948,7 +957,7 @@ ipread_s32bu_p32bu_2(File, %%% provide a higher-lever interface to files. -spec consult(Filename) -> {ok, Terms} | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Terms :: [term()], Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -965,10 +974,10 @@ consult(File) -> -spec path_consult(Path, Filename) -> {ok, Terms, FullName} | {error, Reason} when Path :: [Dir], - Dir :: name(), - Filename :: name(), + Dir :: name_all(), + Filename :: name_all(), Terms :: [term()], - FullName :: filename(), + FullName :: filename_all(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -988,7 +997,7 @@ path_consult(Path, File) -> end. -spec eval(Filename) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -996,7 +1005,7 @@ eval(File) -> eval(File, erl_eval:new_bindings()). -spec eval(Filename, Bindings) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Bindings :: erl_eval:binding_struct(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -1012,9 +1021,9 @@ eval(File, Bs) -> end. -spec path_eval(Path, Filename) -> {ok, FullName} | {error, Reason} when - Path :: [Dir :: name()], - Filename :: name(), - FullName :: filename(), + Path :: [Dir :: name_all()], + Filename :: name_all(), + FullName :: filename_all(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -1023,10 +1032,10 @@ path_eval(Path, File) -> -spec path_eval(Path, Filename, Bindings) -> {ok, FullName} | {error, Reason} when - Path :: [Dir :: name()], - Filename :: name(), + Path :: [Dir :: name_all()], + Filename :: name_all(), Bindings :: erl_eval:binding_struct(), - FullName :: filename(), + FullName :: filename_all(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -1046,7 +1055,7 @@ path_eval(Path, File, Bs) -> end. -spec script(Filename) -> {ok, Value} | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Value :: term(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -1055,7 +1064,7 @@ script(File) -> script(File, erl_eval:new_bindings()). -spec script(Filename, Bindings) -> {ok, Value} | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Bindings :: erl_eval:binding_struct(), Value :: term(), Reason :: posix() | badarg | terminated | system_limit @@ -1073,10 +1082,10 @@ script(File, Bs) -> -spec path_script(Path, Filename) -> {ok, Value, FullName} | {error, Reason} when - Path :: [Dir :: name()], - Filename :: name(), + Path :: [Dir :: name_all()], + Filename :: name_all(), Value :: term(), - FullName :: filename(), + FullName :: filename_all(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -1085,11 +1094,11 @@ path_script(Path, File) -> -spec path_script(Path, Filename, Bindings) -> {ok, Value, FullName} | {error, Reason} when - Path :: [Dir :: name()], - Filename :: name(), + Path :: [Dir :: name_all()], + Filename :: name_all(), Bindings :: erl_eval:binding_struct(), Value :: term(), - FullName :: filename(), + FullName :: filename_all(), Reason :: posix() | badarg | terminated | system_limit | {Line :: integer(), Mod :: module(), Term :: term()}. @@ -1118,11 +1127,11 @@ path_script(Path, File, Bs) -> -spec path_open(Path, Filename, Modes) -> {ok, IoDevice, FullName} | {error, Reason} when - Path :: [Dir :: name()], - Filename :: name(), + Path :: [Dir :: name_all()], + Filename :: name_all(), Modes :: [mode()], IoDevice :: io_device(), - FullName :: filename(), + FullName :: filename_all(), Reason :: posix() | badarg | system_limit. path_open(PathList, Name, Mode) -> @@ -1144,7 +1153,7 @@ path_open(PathList, Name, Mode) -> end. -spec change_mode(Filename, Mode) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Mode :: integer(), Reason :: posix() | badarg. @@ -1153,7 +1162,7 @@ change_mode(Name, Mode) write_file_info(Name, #file_info{mode=Mode}). -spec change_owner(Filename, Uid) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Uid :: integer(), Reason :: posix() | badarg. @@ -1162,7 +1171,7 @@ change_owner(Name, OwnerId) write_file_info(Name, #file_info{uid=OwnerId}). -spec change_owner(Filename, Uid, Gid) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Uid :: integer(), Gid :: integer(), Reason :: posix() | badarg. @@ -1172,7 +1181,7 @@ change_owner(Name, OwnerId, GroupId) write_file_info(Name, #file_info{uid=OwnerId, gid=GroupId}). -spec change_group(Filename, Gid) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Gid :: integer(), Reason :: posix() | badarg. @@ -1181,7 +1190,7 @@ change_group(Name, GroupId) write_file_info(Name, #file_info{gid=GroupId}). -spec change_time(Filename, Mtime) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Mtime :: date_time(), Reason :: posix() | badarg. @@ -1191,7 +1200,7 @@ change_time(Name, {{Y, M, D}, {H, Min, Sec}}=Time) write_file_info(Name, #file_info{mtime=Time}). -spec change_time(Filename, Atime, Mtime) -> ok | {error, Reason} when - Filename :: name(), + Filename :: name_all(), Atime :: date_time(), Mtime :: date_time(), Reason :: posix() | badarg. @@ -1239,7 +1248,7 @@ sendfile(File, Sock, Offset, Bytes, Opts) -> -spec sendfile(Filename, Socket) -> {'ok', non_neg_integer()} | {'error', inet:posix() | closed | badarg | not_owner} - when Filename :: name(), + when Filename :: name_all(), Socket :: inet:socket(). sendfile(Filename, Sock) -> case file:open(Filename, [read, raw, binary]) of diff --git a/lib/kernel/src/file_server.erl b/lib/kernel/src/file_server.erl index 73202319b9..49ec6f96cc 100644 --- a/lib/kernel/src/file_server.erl +++ b/lib/kernel/src/file_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2011. All Rights Reserved. +%% Copyright Ericsson AB 2000-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -170,7 +170,7 @@ handle_call({read_link_info, Name, Opts}, _From, Handle) -> handle_call({read_link, Name}, _From, Handle) -> {reply, ?PRIM_FILE:read_link(Handle, Name), Handle}; handle_call({read_link_all, Name}, _From, Handle) -> - {reply, ?PRIM_FILE:read_link(Handle, Name), Handle}; + {reply, ?PRIM_FILE:read_link_all(Handle, Name), Handle}; handle_call({make_link, Old, New}, _From, Handle) -> {reply, ?PRIM_FILE:make_link(Handle, Old, New), Handle}; diff --git a/lib/kernel/src/gen_tcp.erl b/lib/kernel/src/gen_tcp.erl index 23867300a5..a98ed4c238 100644 --- a/lib/kernel/src/gen_tcp.erl +++ b/lib/kernel/src/gen_tcp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index 06d404905d..e676ca997d 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -218,7 +218,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> {MFAs,Addresses} = exports(ExportMap, CodeAddress), %% Remove references to old versions of the module. ReferencesToPatch = get_refs_from(MFAs, []), - remove_refs_from(MFAs), + ok = remove_refs_from(MFAs), %% Patch all dynamic references in the code. %% Function calls, Atoms, Constants, System calls patch(Refs, CodeAddress, ConstMap2, Addresses, TrampolineMap), @@ -802,7 +802,7 @@ patch_to_emu_step1(Mod) -> %% Find all call sites that call these MFAs. As a side-effect, %% create native stubs for any MFAs that are referred. ReferencesToPatch = get_refs_from(MFAs, []), - remove_refs_from(MFAs), + ok = remove_refs_from(MFAs), ReferencesToPatch; false -> %% The first time we load the module, no redirection needs to be done. @@ -846,11 +846,8 @@ get_refs_from(MFAs, []) -> mark_referred_from(MFAs), MFAs. -mark_referred_from([MFA|MFAs]) -> - hipe_bifs:mark_referred_from(MFA), - mark_referred_from(MFAs); -mark_referred_from([]) -> - []. +mark_referred_from(MFAs) -> + lists:foreach(fun(MFA) -> hipe_bifs:mark_referred_from(MFA) end, MFAs). %%-------------------------------------------------------------------- %% Given a list of MFAs with referred_from references, update their @@ -858,11 +855,8 @@ mark_referred_from([]) -> %% %% The {MFA,Refs} list must come from get_refs_from/2. %% -redirect([MFA|Rest]) -> - hipe_bifs:redirect_referred_from(MFA), - redirect(Rest); -redirect([]) -> - ok. +redirect(MFAs) -> + lists:foreach(fun(MFA) -> hipe_bifs:redirect_referred_from(MFA) end, MFAs). %%-------------------------------------------------------------------- %% Given a list of MFAs, remove all referred_from references having @@ -874,11 +868,8 @@ redirect([]) -> %% list. The refers_to list is used here to find the CalleeMFAs whose %% referred_from lists should be updated. %% -remove_refs_from([CallerMFA|CallerMFAs]) -> - hipe_bifs:remove_refs_from(CallerMFA), - remove_refs_from(CallerMFAs); -remove_refs_from([]) -> - []. +remove_refs_from(MFAs) -> + lists:foreach(fun(MFA) -> hipe_bifs:remove_refs_from(MFA) end, MFAs). %%-------------------------------------------------------------------- diff --git a/lib/kernel/src/inet6_tcp_dist.erl b/lib/kernel/src/inet6_tcp_dist.erl index 2315a56582..2cb0e10c87 100644 --- a/lib/kernel/src/inet6_tcp_dist.erl +++ b/lib/kernel/src/inet6_tcp_dist.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl index 70f3c87723..8005eff58c 100644 --- a/lib/kernel/src/inet_tcp_dist.erl +++ b/lib/kernel/src/inet_tcp_dist.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 4c040f0a0e..fc17db2745 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -716,7 +716,7 @@ analyse([], [This={M,F,A}|Path], Visited, ErrCnt0) -> %% These modules should be loaded by code.erl before %% the code_server is started. OK = [erlang, os, prim_file, erl_prim_loader, init, ets, - code_server, lists, lists_sort, unicode, binary, filename, packages, + code_server, lists, lists_sort, unicode, binary, filename, gb_sets, gb_trees, hipe_unified_loader, hipe_bifs, prim_zip, zlib], ErrCnt1 = @@ -822,6 +822,10 @@ check_funs({'$M_EXPR','$F_EXPR',2}, check_funs({'$M_EXPR','$F_EXPR',1}, [{lists,foreach,2}, {hipe_unified_loader,patch_consts,3} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',1}, + [{lists,foreach,2}, + {hipe_unified_loader,mark_referred_from,1}, + {hipe_unified_loader,get_refs_from,2}| _]) -> 0; check_funs({'$M_EXPR',warning_msg,2}, [{code_server,finish_on_load_report,2} | _]) -> 0; %% This is cheating! /raimo @@ -1590,7 +1594,7 @@ native_early_modules_1(Architecture) -> ?line true = lists:all(fun code:is_module_native/1, [ets,file,filename,gb_sets,gb_trees, %%hipe_unified_loader, no_native as workaround - lists,os,packages]), + lists,os]), ok end. diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index fd4d5bd24e..c604e7073f 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -46,6 +46,7 @@ read_write_file/1, names/1]). -export([cur_dir_0/1, cur_dir_1/1, make_del_dir/1, list_dir/1,list_dir_error/1, + untranslatable_names/1, untranslatable_names_error/1, pos1/1, pos2/1]). -export([close/1, consult1/1, path_consult/1, delete/1]). -export([ eval1/1, path_eval/1, script1/1, path_script/1, @@ -56,7 +57,7 @@ -export([rename/1, access/1, truncate/1, datasync/1, sync/1, read_write/1, pread_write/1, append/1, exclusive/1]). -export([ e_delete/1, e_rename/1, e_make_dir/1, e_del_dir/1]). --export([otp_5814/1]). +-export([otp_5814/1, otp_10852/1]). -export([ read_not_really_compressed/1, read_compressed_cooked/1, read_compressed_cooked_binary/1, @@ -111,13 +112,14 @@ all() -> {group, files}, delete, rename, names, {group, errors}, {group, compression}, {group, links}, copy, delayed_write, read_ahead, segment_read, segment_write, - ipread, pid2name, interleaved_read_write, otp_5814, + ipread, pid2name, interleaved_read_write, otp_5814, otp_10852, large_file, large_write, read_line_1, read_line_2, read_line_3, read_line_4, standard_io]. groups() -> [{dirs, [], [make_del_dir, cur_dir_0, cur_dir_1, - list_dir, list_dir_error]}, + list_dir, list_dir_error, untranslatable_names, + untranslatable_names_error]}, {files, [], [{group, open}, {group, pos}, {group, file_info}, {group, consult}, {group, eval}, {group, script}, @@ -557,6 +559,112 @@ list_dir_1(TestDir, Cnt, Sorted0) -> Sorted = lists:sort(DirList1), list_dir_1(TestDir, Cnt-1, Sorted). +untranslatable_names(Config) -> + case no_untranslatable_names() of + true -> + {skip,"Not a problem on this OS"}; + false -> + untranslatable_names_1(Config) + end. + +untranslatable_names_1(Config) -> + {ok,OldCwd} = file:get_cwd(), + PrivDir = ?config(priv_dir, Config), + Dir = filename:join(PrivDir, "untranslatable_names"), + ok = file:make_dir(Dir), + Node = start_node(untranslatable_names, "+fnu"), + try + ok = file:set_cwd(Dir), + [ok = file:write_file(F, F) || {_,F} <- untranslatable_names()], + + ExpectedListDir0 = [unicode:characters_to_list(N, utf8) || + {utf8,N} <- untranslatable_names()], + ExpectedListDir = lists:sort(ExpectedListDir0), + io:format("ExpectedListDir: ~p\n", [ExpectedListDir]), + ExpectedListDir = call_and_sort(Node, file, list_dir, [Dir]), + + ExpectedListDirAll0 = [case Enc of + utf8 -> + unicode:characters_to_list(N, utf8); + latin1 -> + N + end || {Enc,N} <- untranslatable_names()], + ExpectedListDirAll = lists:sort(ExpectedListDirAll0), + io:format("ExpectedListDirAll: ~p\n", [ExpectedListDirAll]), + ExpectedListDirAll = call_and_sort(Node, file, list_dir_all, [Dir]) + after + catch test_server:stop_node(Node), + file:set_cwd(OldCwd), + [file:delete(F) || {_,F} <- untranslatable_names()], + file:del_dir(Dir) + end, + ok. + +untranslatable_names_error(Config) -> + case no_untranslatable_names() of + true -> + {skip,"Not a problem on this OS"}; + false -> + untranslatable_names_error_1(Config) + end. + +untranslatable_names_error_1(Config) -> + {ok,OldCwd} = file:get_cwd(), + PrivDir = ?config(priv_dir, Config), + Dir = filename:join(PrivDir, "untranslatable_names_error"), + ok = file:make_dir(Dir), + Node = start_node(untranslatable_names, "+fnue"), + try + ok = file:set_cwd(Dir), + [ok = file:write_file(F, F) || {_,F} <- untranslatable_names()], + + ExpectedListDir0 = [unicode:characters_to_list(N, utf8) || + {utf8,N} <- untranslatable_names()], + ExpectedListDir = lists:sort(ExpectedListDir0), + io:format("ExpectedListDir: ~p\n", [ExpectedListDir]), + {error,{no_translation,BadFile}} = + rpc:call(Node, file, list_dir, [Dir]), + true = lists:keymember(BadFile, 2, untranslatable_names()) + + after + catch test_server:stop_node(Node), + file:set_cwd(OldCwd), + [file:delete(F) || {_,F} <- untranslatable_names()], + file:del_dir(Dir) + end, + ok. + +untranslatable_names() -> + [{utf8,<<"abc">>}, + {utf8,<<"def">>}, + {utf8,<<"Lagerl",195,182,"f">>}, + {utf8,<<195,150,"stra Emterwik">>}, + {latin1,<<"M",229,"rbacka">>}, + {latin1,<<"V",228,"rmland">>}]. + +call_and_sort(Node, M, F, A) -> + {ok,Res} = rpc:call(Node, M, F, A), + lists:sort(Res). + +no_untranslatable_names() -> + case os:type() of + {unix,darwin} -> true; + {win32,_} -> true; + _ -> false + end. + +start_node(Name, Args) -> + [_,Host] = string:tokens(atom_to_list(node()), "@"), + ct:log("Trying to start ~w@~s~n", [Name,Host]), + case test_server:start_node(Name, peer, [{args,Args}]) of + {error,Reason} -> + test_server:fail(Reason); + {ok,Node} -> + ct:log("Node ~p started~n", [Node]), + Node + end. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -3410,6 +3518,49 @@ otp_5814(Config) when is_list(Config) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +otp_10852(suite) -> + []; +otp_10852(doc) -> + ["OTP-10852. +fnu and latin1 filenames"]; +otp_10852(Config) when is_list(Config) -> + Node = start_node(erl_pp_helper, "+fnu"), + Dir = ?config(priv_dir, Config), + B = filename:join(Dir, <<"\xE4">>), + ok = rpc_call(Node, get_cwd, [B]), + {error, no_translation} = rpc_call(Node, set_cwd, [B]), + ok = rpc_call(Node, delete, [B]), + ok = rpc_call(Node, rename, [B, B]), + ok = rpc_call(Node, read_file_info, [B]), + ok = rpc_call(Node, read_link_info, [B]), + ok = rpc_call(Node, read_link, [B]), + ok = rpc_call(Node, write_file_info, [B,#file_info{}]), + ok = rpc_call(Node, list_dir, [B]), + ok = rpc_call(Node, list_dir_all, [B]), + ok = rpc_call(Node, read_file, [B]), + ok = rpc_call(Node, make_link, [B,B]), + ok = rpc_call(Node, make_symlink, [B,B]), + ok = rpc_call(Node, delete, [B]), + ok = rpc_call(Node, make_dir, [B]), + ok = rpc_call(Node, del_dir, [B]), + ok = rpc_call(Node, write_file, [B,B]), + {ok, Fd} = rpc_call(Node, open, [B,[read]]), + ok = rpc_call(Node, close, [Fd]), + {ok,0} = rpc_call(Node, copy, [B,B]), + {ok, Fd2, B} = rpc_call(Node, path_open, [["."], B, [read]]), + ok = rpc_call(Node, close, [Fd2]), + true = test_server:stop_node(Node), + ok. + +rpc_call(N, F, As) -> + case rpc:call(N, ?FILE_MODULE, F, As) of + {error, enotsup} -> ok; + {error, enoent} -> ok; + {error, badarg} -> ok; + Else -> Else + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + large_file(suite) -> []; large_file(doc) -> diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index e8529db1dc..a6728564e4 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -3,7 +3,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -340,9 +340,9 @@ check_icky(Mod) -> ?line true=(length("åäö") =:= 3), ?line UniMode = file:native_name_encoding() =/= latin1, ?line make_icky_dir(Mod), - ?line {ok, L0} = Mod:list_dir("."), + {ok, L0} = Mod:list_dir_all("."), ?line L1 = lists:sort(L0), - io:format("~p ~p~n",[L1,list(icky_dir())]), + io:format("~p~n~p~n~n",[L1,lists:sort(list(icky_dir()))]), ?line L1 = lists:sort(convlist(list(icky_dir()))), ?line {ok,D2} = Mod:get_cwd(), ?line true = is_list(D2), @@ -357,7 +357,8 @@ check_icky(Mod) -> ?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,icky_dir()))} || {T,S,Targ} <- icky_dir(), T =:= symlink ], ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], - ?line [ {ok, Targ} = fixlink(Mod:read_link(SymL)) || {SymL,Targ,_} <- Syms ], + [ {ok, Targ} = fixlink(Mod:read_link_all(SymL)) || + {SymL,Targ,_} <- Syms ], ?line chk_cre_dir(Mod,[{directory,"åäö_dir",icky_dir()}]), ?line {ok,BeginAt} = Mod:get_cwd(), ?line true = is_list(BeginAt), @@ -369,7 +370,7 @@ check_icky(Mod) -> ?line ok = Mod:set_cwd(".."), ?line {ok,BeginAt} = Mod:get_cwd(), ?line rm_r2(Mod,"åäö_dir"), - {OS,TYPE} = os:type(), + {OS,_} = os:type(), % Check that treat_icky really converts to the same as the OS case UniMode of true -> @@ -377,7 +378,7 @@ check_icky(Mod) -> ?line ok = Mod:set_cwd("åäö_dir"), ?line ok = Mod:write_file(<<"ååå">>,<<"hello">>), ?line Treated = treat_icky(<<"ååå">>), - ?line {ok,[Treated]} = Mod:list_dir("."), + {ok,[Treated]} = Mod:list_dir_all("."), ?line ok = Mod:delete(<<"ååå">>), ?line {ok,[]} = Mod:list_dir("."), ?line ok = Mod:set_cwd(".."), @@ -393,15 +394,7 @@ check_icky(Mod) -> true -> ok end, - ?line ok = Mod:set_cwd(treat_icky(<<"åäö_dir">>)), - ?line {ok, NowAt2} = Mod:get_cwd(), - io:format("~p~n",[NowAt2]), - % Cannot create raw unicode-breaking filenames on windows or macos - ?line true = ((((not UniMode) or (OS =:= win32) or (TYPE=:=darwin)) and is_list(NowAt2)) orelse ((UniMode) and is_binary(NowAt2))), - ?line true = BeginAt =/= NowAt2, - ?line ok = Mod:set_cwd(".."), ?line {ok,BeginAt} = Mod:get_cwd(), - ?line rm_r2(Mod,conv(treat_icky(<<"åäö_dir">>))), case has_links() of true -> ?line ok = Mod:make_link("fil1","nisseö"), @@ -485,7 +478,7 @@ check_very_icky(Mod) -> ok end, ?line make_very_icky_dir(Mod), - ?line {ok, L0} = Mod:list_dir("."), + {ok, L0} = Mod:list_dir_all("."), ?line L1 = lists:sort(L0), ?line L1 = lists:sort(convlist(list(very_icky_dir()))), ?line {ok,D2} = Mod:get_cwd(), @@ -494,7 +487,8 @@ check_very_icky(Mod) -> ?line Syms = [ {S,conv(Targ),list_to_binary(get_data(Targ,very_icky_dir()))} || {T,S,Targ} <- very_icky_dir(), T =:= symlink ], ?line [ {ok, Cont} = Mod:read_file(SymL) || {SymL,_,Cont} <- Syms ], - ?line [ {ok, Targ} = fixlink(Mod:read_link(SymL)) || {SymL,Targ,_} <- Syms ], + ?line [ {ok, Targ} = fixlink(Mod:read_link_all(SymL)) || + {SymL,Targ,_} <- Syms ], ?line chk_cre_dir(Mod,[{directory,[1088,1079,1091]++"_dir",very_icky_dir()}]), ?line {ok,BeginAt} = Mod:get_cwd(), ?line true = is_list(BeginAt), @@ -559,22 +553,6 @@ check_very_icky(Mod) -> FI#file_info{mode = NewMode2}), ?line {ok,#file_info{mode = NewMode2}} = Mod:read_file_info([956,965,963,954,959,49]), - ?line NumOK0 = case has_links() of - true -> 5; - false -> 3 - end, - ?line NumNOK0 = case has_links() of - true -> 4; - false -> 3 - end, - ?line {NumOK,NumNOK} = case is_binary(treat_icky(<<"foo">>)) of - false -> - {NumOK0+NumNOK0,0}; - true -> - {NumOK0,NumNOK0} - end, - ?line {NumOK,NumNOK} = filelib:fold_files(".",".*",true,fun(_F,{N,M}) when is_list(_F) -> io:format("~ts~n",[_F]),{N+1,M}; (_F,{N,M}) -> io:format("~p~n",[_F]),{N,M+1} end,{0,0}), - ?line ok = filelib:fold_files(".",[1076,1089,1072,124,46,42],true,fun(_F,_) -> ok end,false), ok catch throw:need_unicode_mode -> @@ -593,7 +571,7 @@ check_very_icky(Mod) -> rm_rf(Mod,Dir) -> case Mod:read_link_info(Dir) of {ok, #file_info{type = directory}} -> - {ok, Content} = Mod:list_dir(Dir), + {ok, Content} = Mod:list_dir_all(Dir), [ rm_rf(Mod,filename:join(Dir,C)) || C <- Content ], Mod:del_dir(Dir), ok; @@ -608,7 +586,7 @@ rm_r(Mod,Dir) -> case Mod:read_link_info(Dir) of {ok, #file_info{type = directory}} -> {ok,#file_info{type = directory}} = Mod:read_file_info(Dir), - {ok, Content} = Mod:list_dir(Dir), + {ok, Content} = Mod:list_dir_all(Dir), [ true = is_list(Part) || Part <- Content ], [ true = is_list(filename:join(Dir,Part)) || Part <- Content ], [ rm_r(Mod,filename:join(Dir,C)) || C <- Content ], @@ -626,7 +604,7 @@ rm_r2(Mod,Dir) -> case Mod:read_link_info(Dir) of {ok, #file_info{type = directory}} -> {ok,#file_info{type = directory}} = Mod:read_file_info(Dir), - {ok, Content} = Mod:list_dir(Dir), + {ok, Content} = Mod:list_dir_all(Dir), UniMode = file:native_name_encoding() =/= latin1, [ true = (is_list(Part) orelse UniMode) || Part <- Content ], [ true = (is_list(filename:join(Dir,Part)) orelse UniMode) || Part <- Content ], diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index a72e76f813..6b672004ec 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -50,6 +50,14 @@ oct_acceptor/1, otp_7731_server/1, zombie_server/2, do_iter_max_socks/2]). +init_per_testcase(iter_max_socks, Config) when is_list(Config) -> + Dog = case os:type() of + {win32,_} -> + test_server:timetrap(test_server:minutes(30)); + _Else -> + test_server:timetrap(test_server:seconds(240)) + end, + [{watchdog, Dog}|Config]; init_per_testcase(_Func, Config) when is_list(Config) -> Dog = test_server:timetrap(test_server:seconds(240)), [{watchdog, Dog}|Config]. @@ -590,7 +598,7 @@ iter_max_socks(doc) -> ["Open as many sockets as possible. Do this several times and check ", "that we get the same number of sockets every time."]; iter_max_socks(Config) when is_list(Config) -> - N = 20, + N = case os:type() of {win32,_} -> 10; _ -> 20 end, %% Run on a different node in order to limit the effect if this test fails. Dir = filename:dirname(code:which(?MODULE)), {ok,Node} = test_server:start_node(test_iter_max_socks,slave, diff --git a/lib/kernel/test/prim_file_SUITE.erl b/lib/kernel/test/prim_file_SUITE.erl index 4550cb1770..ac75037536 100644 --- a/lib/kernel/test/prim_file_SUITE.erl +++ b/lib/kernel/test/prim_file_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2011. All Rights Reserved. +%% Copyright Ericsson AB 2000-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index b6cf4407d2..49404196dd 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 2.16.1 +KERNEL_VSN = 2.16.2 diff --git a/lib/megaco/doc/src/notes.xml b/lib/megaco/doc/src/notes.xml index a092946fef..bb30ce4c2a 100644 --- a/lib/megaco/doc/src/notes.xml +++ b/lib/megaco/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2000</year><year>2012</year> + <year>2000</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/megaco/vsn.mk b/lib/megaco/vsn.mk index a8f9c5917a..b61cf2102c 100644 --- a/lib/megaco/vsn.mk +++ b/lib/megaco/vsn.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 6e13331508..5942a40a87 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1996</year><year>2012</year> + <year>1996</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/observer/doc/src/notes.xml b/lib/observer/doc/src/notes.xml index 44fe2727bc..ddf3b3fe37 100644 --- a/lib/observer/doc/src/notes.xml +++ b/lib/observer/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/odbc/doc/src/notes.xml b/lib/odbc/doc/src/notes.xml index 0110aa8e81..792dc3fddd 100644 --- a/lib/odbc/doc/src/notes.xml +++ b/lib/odbc/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,7 +31,24 @@ <p>This document describes the changes made to the odbc application. </p> - <section><title>ODBC 2.10.14</title> + <section><title>ODBC 2.10.15</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fixed calling odbc:param_query/3 and odbc:param_query/4 + with unparameterized query string and empty parameters + list. Thanks to Danil Onishchenko.</p> + <p> + Own Id: OTP-10798</p> + </item> + </list> + </section> + +</section> + +<section><title>ODBC 2.10.14</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/odbc/src/odbc.erl b/lib/odbc/src/odbc.erl index dde96907e5..ebfcd20d9a 100644 --- a/lib/odbc/src/odbc.erl +++ b/lib/odbc/src/odbc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% Copyright Ericsson AB 1999-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/odbc/test/odbc_connect_SUITE.erl b/lib/odbc/test/odbc_connect_SUITE.erl index b06384fc94..74ae2c96e6 100644 --- a/lib/odbc/test/odbc_connect_SUITE.erl +++ b/lib/odbc/test/odbc_connect_SUITE.erl @@ -277,13 +277,19 @@ port_dies(_Config) -> {ok, Ref} = odbc:connect(?RDBMS:connection_string(), odbc_test_lib:platform_options()), {status, _} = process_info(Ref, status), process_flag(trap_exit, true), - Port = lists:last(erlang:ports()), - exit(Port, kill), - %% Wait for exit_status from port 5000 ms (will not get a exit - %% status in this case), then wait a little longer to make sure - %% the port and the controlprocess has had time to terminate. - test_server:sleep(10000), - undefined = process_info(Ref, status). + NamedPorts = [{P, erlang:port_info(P, name)} || P <- erlang:ports()], + case [P || {P, {name, Name}} <- NamedPorts, is_odbcserver(Name)] of + [Port] -> + exit(Port, kill), + %% Wait for exit_status from port 5000 ms (will not get a exit + %% status in this case), then wait a little longer to make sure + %% the port and the controlprocess has had time to terminate. + test_server:sleep(10000), + undefined = process_info(Ref, status); + [] -> + ct:fail([erlang:port_info(P, name) || P <- erlang:ports()]) + end. + %%------------------------------------------------------------------------- control_process_dies(doc) -> @@ -292,13 +298,17 @@ control_process_dies(suite) -> []; control_process_dies(_Config) -> {ok, Ref} = odbc:connect(?RDBMS:connection_string(), odbc_test_lib:platform_options()), process_flag(trap_exit, true), - Port = lists:last(erlang:ports()), - {connected, Ref} = erlang:port_info(Port, connected), - exit(Ref, kill), - test_server:sleep(500), - undefined = erlang:port_info(Port, connected). - %% Check for c-program still running, how? - + NamedPorts = [{P, erlang:port_info(P, name)} || P <- erlang:ports()], + case [P || {P, {name, Name}} <- NamedPorts, is_odbcserver(Name)] of + [Port] -> + {connected, Ref} = erlang:port_info(Port, connected), + exit(Ref, kill), + test_server:sleep(500), + undefined = erlang:port_info(Port, connected); + %% Check for c-program still running, how? + [] -> + ct:fail([erlang:port_info(P, name) || P <- erlang:ports()]) + end. %%------------------------------------------------------------------------- client_dies_normal(doc) -> @@ -868,3 +878,13 @@ extended_errors(Config) when is_list(Config)-> ok = odbc:disconnect(Ref), ok = odbc:disconnect(RefExtended). + + +is_odbcserver(Name) -> + case re:run(Name, "odbcserver") of + {match, _} -> + true; + _ -> + false + end. + diff --git a/lib/odbc/test/odbc_query_SUITE.erl b/lib/odbc/test/odbc_query_SUITE.erl index 61253b29aa..062373afa0 100644 --- a/lib/odbc/test/odbc_query_SUITE.erl +++ b/lib/odbc/test/odbc_query_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2011. All Rights Reserved. +%% Copyright Ericsson AB 2002-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/orber/doc/src/notes.xml b/lib/orber/doc/src/notes.xml index ceb25dd2c0..d43ab3ac24 100644 --- a/lib/orber/doc/src/notes.xml +++ b/lib/orber/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2012</year> + <year>1997</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/os_mon/doc/src/notes.xml b/lib/os_mon/doc/src/notes.xml index 5bc6ce1406..61eac937e1 100644 --- a/lib/os_mon/doc/src/notes.xml +++ b/lib/os_mon/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/otp_mibs/doc/src/notes.xml b/lib/otp_mibs/doc/src/notes.xml index 04a13f47c2..8986eab9ad 100644 --- a/lib/otp_mibs/doc/src/notes.xml +++ b/lib/otp_mibs/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/parsetools/doc/src/notes.xml b/lib/parsetools/doc/src/notes.xml index 72b0280da1..a0816dc728 100644 --- a/lib/parsetools/doc/src/notes.xml +++ b/lib/parsetools/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2011</year> + <year>1997</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,20 @@ </header> <p>This document describes the changes made to the Parsetools application.</p> +<section><title>Parsetools 2.0.9</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Miscellaneous updates due to Unicode support. </p> + <p> + Own Id: OTP-10820</p> + </item> + </list> + </section> + +</section> + <section><title>Parsetools 2.0.8</title> <section><title>Improvements and New Features</title> diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl index 22b496258f..e531b78a5b 100644 --- a/lib/parsetools/src/leex.erl +++ b/lib/parsetools/src/leex.erl @@ -127,7 +127,7 @@ file(File, Opts0) -> leex_ret(St). format_error({file_error, Reason}) -> - io_lib:fwrite("~s",[file:format_error(Reason)]); + io_lib:fwrite("~ts",[file:format_error(Reason)]); format_error(missing_defs) -> "missing Definitions"; format_error(missing_rules) -> "missing Rules"; format_error(missing_code) -> "missing Erlang code"; @@ -301,10 +301,10 @@ pack_warnings([]) -> report_errors(St) -> when_opt(fun () -> foreach(fun({File,{none,Mod,E}}) -> - io:fwrite("~s: ~ts\n", + io:fwrite("~ts: ~ts\n", [File,Mod:format_error(E)]); ({File,{Line,Mod,E}}) -> - io:fwrite("~s:~w: ~ts\n", + io:fwrite("~ts:~w: ~ts\n", [File,Line,Mod:format_error(E)]) end, sort(St#leex.errors)) end, report_errors, St#leex.opts). @@ -319,11 +319,11 @@ report_warnings(St) -> ShouldReport = member(report_warnings, St#leex.opts) orelse ReportWerror, when_bool(fun () -> foreach(fun({File,{none,Mod,W}}) -> - io:fwrite("~s: ~s~ts\n", + io:fwrite("~ts: ~s~ts\n", [File,Prefix, Mod:format_error(W)]); ({File,{Line,Mod,W}}) -> - io:fwrite("~s:~w: ~s~ts\n", + io:fwrite("~ts:~w: ~s~ts\n", [File,Line,Prefix, Mod:format_error(W)]) end, sort(St#leex.warnings)) @@ -401,7 +401,7 @@ parse_file(St0) -> {ok,Xfile} -> St1 = St0#leex{encoding = epp:set_encoding(Xfile)}, try - verbose_print(St1, "Parsing file ~s, ", [St1#leex.xfile]), + verbose_print(St1, "Parsing file ~ts, ", [St1#leex.xfile]), %% We KNOW that errors throw so we can ignore them here. {ok,Line1,St2} = parse_head(Xfile, St1), {ok,Line2,Macs,St3} = parse_defs(Xfile, Line1, St2), @@ -1292,7 +1292,7 @@ pack_dfa([], _, Rs, PDFA) -> {PDFA,Rs}. %% the code for the actions. out_file(St0, DFA, DF, Actions, Code) -> - verbose_print(St0, "Writing file ~s, ", [St0#leex.efile]), + verbose_print(St0, "Writing file ~ts, ", [St0#leex.efile]), case open_inc_file(St0) of {ok,Ifile} -> try @@ -1582,7 +1582,7 @@ pp_sep(_, _, _, _) -> " ". %% with Graphviz. out_dfa_graph(St, DFA, DF) -> - verbose_print(St, "Writing DFA to file ~s, ", [St#leex.gfile]), + verbose_print(St, "Writing DFA to file ~ts, ", [St#leex.gfile]), case file:open(St#leex.gfile, [write]) of {ok,Gfile} -> try @@ -1644,7 +1644,7 @@ output_encoding_comment(File, #leex{encoding = Encoding}) -> io:fwrite(File, <<"%% ~s\n">>, [epp:encoding_to_string(Encoding)]). output_file_directive(File, Filename, Line) -> - io:fwrite(File, <<"-file(~s, ~w).\n">>, + io:fwrite(File, <<"-file(~ts, ~w).\n">>, [format_filename(Filename), Line]). format_filename(Filename) -> diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl index 30e0db421e..53292b037a 100644 --- a/lib/parsetools/src/yecc.erl +++ b/lib/parsetools/src/yecc.erl @@ -185,7 +185,7 @@ format_error({endsymbol_is_terminal, Symbol}) -> format_error({error, Module, Error}) -> Module:format_error(Error); format_error({file_error, Reason}) -> - io_lib:fwrite("~s",[file:format_error(Reason)]); + io_lib:fwrite("~ts",[file:format_error(Reason)]); format_error(illegal_empty) -> io_lib:fwrite("illegal use of empty symbol", []); format_error({internal_error, Error}) -> @@ -481,7 +481,7 @@ generate(St0) -> ?PASS(action_conflicts), ?PASS(write_file)], F = case member(time, St1#yecc.options) of true -> - io:fwrite(<<"Generating parser from grammar in ~s\n">>, + io:fwrite(<<"Generating parser from grammar in ~ts\n">>, [format_filename(St1#yecc.infile)]), fun timeit/3; false -> @@ -858,10 +858,10 @@ report_errors(St) -> case member(report_errors, St#yecc.options) of true -> foreach(fun({File,{none,Mod,E}}) -> - io:fwrite(<<"~s: ~ts\n">>, + io:fwrite(<<"~ts: ~ts\n">>, [File,Mod:format_error(E)]); ({File,{Line,Mod,E}}) -> - io:fwrite(<<"~s:~w: ~ts\n">>, + io:fwrite(<<"~ts:~w: ~ts\n">>, [File,Line,Mod:format_error(E)]) end, sort(St#yecc.errors)); false -> @@ -878,11 +878,11 @@ report_warnings(St) -> case member(report_warnings, St#yecc.options) orelse ReportWerror of true -> foreach(fun({File,{none,Mod,W}}) -> - io:fwrite(<<"~s: ~s~ts\n">>, + io:fwrite(<<"~ts: ~s~ts\n">>, [File,Prefix, Mod:format_error(W)]); ({File,{Line,Mod,W}}) -> - io:fwrite(<<"~s:~w: ~s~ts\n">>, + io:fwrite(<<"~ts:~w: ~s~ts\n">>, [File,Line,Prefix, Mod:format_error(W)]) end, sort(St#yecc.warnings)); @@ -2518,7 +2518,7 @@ output_encoding_comment(#yecc{encoding = Encoding}=St) -> fwrite(St, <<"%% ~s\n">>, [epp:encoding_to_string(Encoding)]). output_file_directive(St, Filename, Line) when St#yecc.file_attrs -> - fwrite(St, <<"-file(~s, ~w).\n">>, + fwrite(St, <<"-file(~ts, ~w).\n">>, [format_filename(Filename), Line]); output_file_directive(St, _Filename, _Line) -> St. diff --git a/lib/parsetools/vsn.mk b/lib/parsetools/vsn.mk index 7874045c09..444caf44a1 100644 --- a/lib/parsetools/vsn.mk +++ b/lib/parsetools/vsn.mk @@ -1 +1 @@ -PARSETOOLS_VSN = 2.0.8 +PARSETOOLS_VSN = 2.0.9 diff --git a/lib/pman/doc/src/notes.xml b/lib/pman/doc/src/notes.xml index 0cd46b0cee..26f966fd86 100644 --- a/lib/pman/doc/src/notes.xml +++ b/lib/pman/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,23 @@ </header> <p>This document describes the changes made to the Pman application.</p> +<section><title>Pman 2.7.1.4</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Tuple funs (deprecated in R15B) are no longer supported.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-10170</p> + </item> + </list> + </section> + +</section> + <section><title>Pman 2.7.1.3</title> <section><title>Improvements and New Features</title> diff --git a/lib/pman/src/pman_win.erl b/lib/pman/src/pman_win.erl index a6ef6fd13e..aec7dc4412 100644 --- a/lib/pman/src/pman_win.erl +++ b/lib/pman/src/pman_win.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/pman/vsn.mk b/lib/pman/vsn.mk index 12ee4b1669..92fb9df222 100644 --- a/lib/pman/vsn.mk +++ b/lib/pman/vsn.mk @@ -1 +1 @@ -PMAN_VSN = 2.7.1.3 +PMAN_VSN = 2.7.1.4 diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index 09b6c79acd..3a7ccbe568 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2008</year> - <year>2012</year> + <year>2013</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> diff --git a/lib/reltool/doc/src/notes.xml b/lib/reltool/doc/src/notes.xml index 69e499723f..a9aed2b0a3 100644 --- a/lib/reltool/doc/src/notes.xml +++ b/lib/reltool/doc/src/notes.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2009</year> - <year>2012</year> + <year>2013</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -37,7 +37,22 @@ thus constitutes one section in this document. The title of each section is the version number of Reltool.</p> - <section><title>Reltool 0.6.2</title> + <section><title>Reltool 0.6.3</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Some updates are made to reltool for handling unicode.</p> + <p> + Own Id: OTP-10781</p> + </item> + </list> + </section> + +</section> + +<section><title>Reltool 0.6.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/reltool/examples/display_args b/lib/reltool/examples/display_args index bf0994ab7c..a8882f7bda 100644 --- a/lib/reltool/examples/display_args +++ b/lib/reltool/examples/display_args @@ -1,8 +1,9 @@ #!/usr/bin/env escript %% -*- erlang -*- -%%! -smp disable +%%! -emuarg emuvalue main(Args) -> + {ok,[Emuvalue]} = init:get_argument(emuarg), io:format("Root dir: ~s\n", [code:root_dir()]), io:format("Script args: ~p\n", [Args]), - io:format("Smp: ~p\n", [erlang:system_info(smp_support)]). + io:format("Emuarg: ~p\n", [Emuvalue]). diff --git a/lib/reltool/src/reltool.erl b/lib/reltool/src/reltool.erl index 2bdf222aa0..8ab2c2399e 100644 --- a/lib/reltool/src/reltool.erl +++ b/lib/reltool/src/reltool.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -120,7 +120,7 @@ apply_fun(Pid, false, Fun) -> apply_fun(Pid, true, Fun) -> case get_status(Pid) of {ok, Warnings} -> - [io:format("~p: ~s\n", [?APPLICATION, W]) || W <- Warnings], + [io:format("~w: ~ts\n", [?APPLICATION, W]) || W <- Warnings], apply_fun(Pid, false, Fun); {error, _Reason} = Error -> stop(Pid), diff --git a/lib/reltool/src/reltool_app_win.erl b/lib/reltool/src/reltool_app_win.erl index 6cd0d2f90b..81ab1687fb 100644 --- a/lib/reltool/src/reltool_app_win.erl +++ b/lib/reltool/src/reltool_app_win.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -173,7 +173,7 @@ loop(#state{xref_pid = Xref, common = C, app = App} = S) -> S#state.mod_wins)}, ?MODULE:loop(S2); Msg -> - error_logger:format("~p~p got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~p\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end. @@ -181,7 +181,7 @@ loop(#state{xref_pid = Xref, common = C, app = App} = S) -> exit_warning({'EXIT', _Pid, shutdown}) -> ok; exit_warning({'EXIT', _Pid, _Reason} = Msg) -> - error_logger:format("~p~p got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~p\n", [?MODULE, self(), Msg]). create_window(#state{app = App} = S) -> @@ -627,7 +627,7 @@ handle_event(#state{sys = Sys, app = App} = S, Wx) -> Items = reltool_utils:get_items(ListCtrl), handle_mod_button(S, Items, Action); _ -> - error_logger:format("~p~p got unexpected app event from " + error_logger:format("~w~w got unexpected app event from " "wx:\n\t~p\n", [?MODULE, self(), Wx]), S @@ -674,8 +674,8 @@ move_mod(App, {_ItemNo, ModStr}, Action) -> blacklist_del -> undefined; _ -> - error_logger:format("~p~p got unexpected mod " - "button event: ~p\n\t ~p\n", + error_logger:format("~w~w got unexpected mod " + "button event: ~w\n\t ~p\n", [?MODULE, self(), ModName, Action]), M#mod.incl_cond end, diff --git a/lib/reltool/src/reltool_fgraph_win.erl b/lib/reltool/src/reltool_fgraph_win.erl index b0deb1bab2..66bc2b5ab3 100644 --- a/lib/reltool/src/reltool_fgraph_win.erl +++ b/lib/reltool/src/reltool_fgraph_win.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -525,7 +525,7 @@ loop(S, G) -> exit(Reason); Other -> - error_logger:format("~p~p got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~p\n", [?MODULE, self(), Other]), loop(S, G) end. diff --git a/lib/reltool/src/reltool_mod_win.erl b/lib/reltool/src/reltool_mod_win.erl index 899423bb6d..b0193a2ae4 100644 --- a/lib/reltool/src/reltool_mod_win.erl +++ b/lib/reltool/src/reltool_mod_win.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -130,7 +130,7 @@ do_init(Parent, WxEnv, Xref, RelPid, C, ModName) -> loop(#state{xref_pid = Xref, common = C, mod = Mod} = S) -> receive Msg -> - %% io:format("~s~p -> ~p\n", [S#state.name, self(), Msg]), + %% io:format("~ts~w -> ~p\n", [S#state.name, self(), Msg]), case Msg of {system, From, SysMsg} -> Dbg = C#common.sys_debug, @@ -170,7 +170,7 @@ loop(#state{xref_pid = Xref, common = C, mod = Mod} = S) -> S2 = handle_event(S, Wx), ?MODULE:loop(S2); _ -> - error_logger:format("~p~p got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~p\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end @@ -335,23 +335,37 @@ find_regular_bin(App, Mod) -> SrcDir = filename:join([ActiveDir, "src"]), ModStr = atom_to_list(Mod#mod.name), Base = "^" ++ ModStr ++ "\\.erl$", - Find = fun(F, _Acc) -> throw(file:read_file(F)) end, + Find = fun(F, _Acc) -> throw({file:read_file(F),epp:read_encoding(F)}) end, case catch filelib:fold_files(SrcDir, Base, true, Find, {error, enoent}) of - {ok, Bin} -> - Bin; + {{ok, Bin},Encoding0} -> + Encoding = + case Encoding0 of + none -> epp:default_encoding(); + _ -> Encoding0 + end, + unicode:characters_to_binary(Bin,Encoding,utf8); {error, enoent} -> %% Reconstructing the source code from debug info if possible BeamFile = filename:join([ActiveDir, "ebin", ModStr ++ ".beam"]), - case beam_lib:chunks(BeamFile, [abstract_code]) of - {ok,{_,[{abstract_code,{_,AC}}]}} -> - IoList = erl_prettypr:format(erl_syntax:form_list(AC)), - list_to_binary(IoList); - _ -> - list_to_binary(["%% Bad luck, cannot find any " - "debug info in the file \"", BeamFile]) + case source_from_beam(BeamFile) of + {ok,Source} -> + Source; + error -> + unicode:characters_to_binary( + ["%% Bad luck, cannot find any " + "debug info in the file \"", BeamFile]) end end. +source_from_beam(Beam) -> + case beam_lib:chunks(Beam, [abstract_code]) of + {ok,{_,[{abstract_code,{_,AC}}]}} -> + IoList = [erl_pp:form(F,[{encoding,utf8}]) || F <- AC], + {ok,unicode:characters_to_binary(IoList)}; + _ -> + error + end. + find_escript_bin(#app{active_dir = ActiveDir}, Mod) -> NotFound = false, ModName = Mod#mod.name, @@ -366,16 +380,10 @@ find_escript_bin(#app{active_dir = ActiveDir}, Mod) -> case beam_lib:version(Bin) of {ok,{M, _}} when M =:= ModName; FullName =:= "." -> - case beam_lib:chunks(Bin, - [abstract_code]) of - {ok,{_,[{abstract_code,{_,AC}}]}} -> - Form = - erl_syntax:form_list(AC), - IoList = - erl_prettypr:format(Form), - {obj, - list_to_binary(IoList)}; - _ -> + case source_from_beam(Bin) of + {ok,Source} -> + {obj,Source}; + error -> Acc end; _ -> @@ -396,12 +404,9 @@ find_escript_bin(#app{active_dir = ActiveDir}, Mod) -> case filename:split(FullName) of [_AppName, "ebin", F] when F =:= ObjFile, Acc =:= NotFound -> - case beam_lib:chunks(GetBin(), - [abstract_code]) of - {ok,{_,[{abstract_code,{_,AC}}]}} -> - Form = erl_syntax:form_list(AC), - IoList = erl_prettypr:format(Form), - {obj, list_to_binary(IoList)}; + case source_from_beam(GetBin()) of + {ok,Source} -> + {obj,Source}; _ -> Acc end; @@ -420,13 +425,15 @@ find_escript_bin(#app{active_dir = ActiveDir}, Mod) -> {ok, {obj, Bin}} -> Bin; _ -> - list_to_binary(["%% Bad luck, cannot find the " - "code in the escript ", Escript, "."]) + unicode:characters_to_binary( + ["%% Bad luck, cannot find the " + "code in the escript ", Escript, "."]) end catch throw:Reason when is_list(Reason) -> - list_to_binary(["%% Bad luck, cannot find the code " - "in the escript ", Escript, ": ", Reason]) + unicode:characters_to_binary( + ["%% Bad luck, cannot find the code " + "in the escript ", Escript, ": ", Reason]) end. create_config_page(S) -> @@ -478,7 +485,7 @@ handle_event(#state{xref_pid = Xref} = S, Wx) -> wxWindow:setFocus(ObjRef), S; _ -> - error_logger:format("~p~p got unexpected mod event from " + error_logger:format("~w~w got unexpected mod event from " "wx:\n\t~p\n", [?MODULE, self(), Wx]), S diff --git a/lib/reltool/src/reltool_server.erl b/lib/reltool/src/reltool_server.erl index c56e29152d..5e25f22a6f 100644 --- a/lib/reltool/src/reltool_server.erl +++ b/lib/reltool/src/reltool_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -399,12 +399,12 @@ loop(#state{sys = Sys} = S) -> {'EXIT', Pid, Reason} when Pid =:= S#state.parent_pid -> exit(Reason); {call, ReplyTo, Ref, Msg} when is_pid(ReplyTo), is_reference(Ref) -> - error_logger:format("~p~p got unexpected call:\n\t~p\n", + error_logger:format("~w~w got unexpected call:\n\t~p\n", [?MODULE, self(), Msg]), reltool_utils:reply(ReplyTo, Ref, {error, {invalid_call, Msg}}), ?MODULE:loop(S); Msg -> - error_logger:format("~p~p got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~p\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end. @@ -422,7 +422,7 @@ do_set_apps(#state{sys = Sys} = S, ChangedApps) -> %% 2) removing #app records if no configurable fields are set %% 3) keeping #app records that are not changed app_update_config([#app{name=Name,is_escript={inlined,Escript}}|_],_SysApps) -> - reltool_utils:throw_error("Application ~p is inlined in ~p. Can not change " + reltool_utils:throw_error("Application ~w is inlined in ~w. Can not change " "configuration for an inlined application.", [Name,Escript]); app_update_config([Config|Configs],SysApps) -> @@ -580,8 +580,8 @@ apps_in_rel(#rel{name = RelName, rel_apps = RelApps}, Apps) -> IA; false -> reltool_utils:throw_error( - "Release ~p uses non existing " - "application ~p", + "Release ~tp uses non existing " + "application ~w", [RelName,RA#rel_app.name]) end; IA -> @@ -602,7 +602,7 @@ more_apps_in_rels([{RelName, AppName} = RA | RelApps], Apps, Acc) -> more_apps_in_rels(RelApps, Apps, Acc2); false -> reltool_utils:throw_error( - "Release ~p uses non existing application ~p", + "Release ~tp uses non existing application ~w", [RelName,AppName]) end end; @@ -640,7 +640,7 @@ app_init_is_included(#state{app_tab = AppTab, mod_tab = ModTab, sys=Sys}, {undefined, false, false, Status}; {exclude, [RelName | _]} -> % App is included in at least one rel reltool_utils:throw_error( - "Application ~p is used in release ~p and cannot be excluded", + "Application ~w is used in release ~tp and cannot be excluded", [AppName,RelName]); {derived, []} -> {undefined, undefined, undefined, Status}; @@ -665,7 +665,7 @@ app_init_is_included(#state{app_tab = AppTab, mod_tab = ModTab, sys=Sys}, Status3. mod_init_is_included(ModTab, M, ModCond, AppCond, Default, Status) -> - %% print(M#mod.name, hipe, "incl_cond -> ~p\n", [AppCond]), + %% print(M#mod.name, hipe, "incl_cond -> ~w\n", [AppCond]), IsIncl = case AppCond of include -> @@ -677,7 +677,7 @@ mod_init_is_included(ModTab, M, ModCond, AppCond, Default, Status) -> derived -> undefined; undefined -> - %% print(M#mod.name, hipe, "mod_cond -> ~p\n", + %% print(M#mod.name, hipe, "mod_cond -> ~w\n", %% [ModCond]), case ModCond of all -> true; @@ -711,23 +711,23 @@ mod_init_is_included(ModTab, M, ModCond, AppCond, Default, Status) -> {false,_} -> ets:insert(ModTab, M2), reltool_utils:add_warning( - "Module ~p exists in applications ~p and ~p. " - "Using module from application ~p.", + "Module ~w exists in applications ~w and ~w. " + "Using module from application ~w.", [M#mod.name, Existing#mod.app_name, M#mod.app_name, M#mod.app_name], Status); {_,false} -> %% Don't insert in ModTab - using Existing reltool_utils:add_warning( - "Module ~p exists in applications ~p and ~p. " - "Using module from application ~p.", + "Module ~w exists in applications ~w and ~w. " + "Using module from application ~w.", [M#mod.name, Existing#mod.app_name, M#mod.app_name,Existing#mod.app_name], Status); {_,_} -> reltool_utils:throw_error( - "Module ~p potentially included by two different " - "applications: ~p and ~p.", + "Module ~w potentially included by two different " + "applications: ~w and ~w.", [M#mod.name,Existing#mod.app_name,M#mod.app_name]) end; [] -> @@ -735,7 +735,7 @@ mod_init_is_included(ModTab, M, ModCond, AppCond, Default, Status) -> Status end, - %% print(M#mod.name, hipe, "~p -> ~p\n", [M2, IsIncl]), + %% print(M#mod.name, hipe, "~p -> ~w\n", [M2, IsIncl]), {M2,Status2}. false_to_undefined(Bool) -> @@ -888,7 +888,7 @@ mod_recap_dependencies(S, A, [#mod{name = ModName}=M1 | Mods], Acc, IsIncl) -> case ets:lookup(S#state.mod_tab, ModName) of [M2] when M2#mod.app_name=:=A#app.name -> ModStatus = do_get_status(M2), - %% print(M2#mod.name, hipe, "status -> ~p\n", [ModStatus]), + %% print(M2#mod.name, hipe, "status -> ~w\n", [ModStatus]), {IsIncl2, M3} = case M2#mod.is_included of true -> @@ -910,8 +910,8 @@ mod_recap_dependencies(S, A, [#mod{name = ModName}=M1 | Mods], Acc, IsIncl) -> %% A module is potensially included by multiple %% applications. This is not allowed! reltool_utils:throw_error( - "Module ~p potentially included by two different applications: " - "~p and ~p", [ModName,A#app.name, " and ", M2#mod.app_name, "."]) + "Module ~w potentially included by two different applications: " + "~w and ~w.", [ModName,A#app.name, M2#mod.app_name]) end; mod_recap_dependencies(_S, _A, [], Acc, IsIncl) -> {lists:reverse(Acc), IsIncl}. @@ -939,7 +939,7 @@ verify_config(#state{app_tab=AppTab, sys=#sys{boot_rel = BootRel, rels = Rels}}, Rels); false -> reltool_utils:throw_error( - "Release ~p is mandatory (used as boot_rel)",[BootRel]) + "Release ~tp is mandatory (used as boot_rel)",[BootRel]) end. check_app(AppTab, {RelName, AppName}, Status) -> @@ -949,7 +949,7 @@ check_app(AppTab, {RelName, AppName}, Status) -> Status; _ -> reltool_utils:throw_error( - "Release ~p uses non included application ~p",[RelName,AppName]) + "Release ~tp uses non included application ~w",[RelName,AppName]) end. check_rel(RelName, RelApps, Status) -> @@ -960,8 +960,8 @@ check_rel(RelName, RelApps, Status) -> Acc; false -> reltool_utils:throw_error( - "Mandatory application ~p is not included in " - "release ~p", [AppName,RelName]) + "Mandatory application ~w is not included in " + "release ~tp", [AppName,RelName]) end end, Mandatory = [kernel, stdlib], @@ -1015,8 +1015,8 @@ refresh_app(#app{name = AppName, lists:foldl( fun(M,S) -> reltool_utils:add_warning( - "Module ~p duplicated in app file for " - "application ~p.", [M, AppName], S) + "Module ~w duplicated in app file for " + "application ~w.", [M, AppName], S) end, Status3, DuplicatedMods) @@ -1074,18 +1074,18 @@ read_app_info(AppFileOrBin, AppFile, AppName, DefaultVsn, Status) -> parse_app_info(AppFile, Info, AI, Status); {ok, _BadApp} -> {missing_app_info(DefaultVsn), - reltool_utils:add_warning("~p: Illegal contents in app file ~p, " + reltool_utils:add_warning("~w: Illegal contents in app file ~tp, " "application tuple with arity 3 expected.", [AppName,AppFile], Status)}; {error, Text} when Text =:= EnoentText -> {missing_app_info(DefaultVsn), - reltool_utils:add_warning("~p: Missing app file ~p.", + reltool_utils:add_warning("~w: Missing app file ~tp.", [AppName,AppFile], Status)}; {error, Text} -> {missing_app_info(DefaultVsn), - reltool_utils:add_warning("~p: Cannot parse app file ~p (~p).", + reltool_utils:add_warning("~w: Cannot parse app file ~tp (~tp).", [AppName,AppFile,Text], Status)} end. @@ -1122,7 +1122,7 @@ parse_app_info(File, [{Key, Val} | KeyVals], AI, Status) -> Status); _ -> Status2 = - reltool_utils:add_warning("Unexpected item ~p in app file ~p.", + reltool_utils:add_warning("Unexpected item ~p in app file ~tp.", [Key,File], Status), parse_app_info(File, KeyVals, AI, Status2) @@ -1212,7 +1212,7 @@ wait_for_processto_die(Ref, Pid, File) -> {'DOWN', Ref, _Type, _Object, _Info} -> ok after timer:seconds(30) -> - error_logger:error_msg("~p(~p): Waiting for process ~p to die ~p\n", + error_logger:error_msg("~w(~w): Waiting for process ~w to die ~tp\n", [?MODULE, ?LINE, Pid, File]), wait_for_processto_die(Ref, Pid, File) end. @@ -1223,7 +1223,7 @@ add_missing_mods(AppName, EbinMods, AppModNames) -> [missing_mod(ModName, AppName) || ModName <- MissingModNames]. missing_mod(ModName, AppName) -> - %% io:format("Missing: ~p -> ~p\n", [AppName, ModName]), + %% io:format("Missing: ~w -> ~w\n", [AppName, ModName]), #mod{name = ModName, app_name = AppName, incl_cond = undefined, @@ -1323,7 +1323,7 @@ read_config(OldSys, Filename) when is_list(Filename) -> {ok, Content} -> reltool_utils:throw_error("Illegal file content: ~p",[Content]); {error, Reason} -> - reltool_utils:throw_error("Illegal config file ~p: ~s", + reltool_utils:throw_error("Illegal config file ~tp: ~ts", [Filename,file:format_error(Reason)]) end; read_config(OldSys, {sys, KeyVals}) -> @@ -1341,7 +1341,7 @@ read_config(OldSys, {sys, KeyVals}) -> NewSys2; false -> reltool_utils:throw_error( - "Release ~p is mandatory (used as boot_rel)", + "Release ~tp is mandatory (used as boot_rel)", [NewSys2#sys.boot_rel]) end; read_config(_OldSys, BadConfig) -> @@ -1510,7 +1510,7 @@ decode(#app{} = App, [{Key, Val} | KeyVals]) -> active_dir = Dir, sorted_dirs = [Dir]}; false -> - reltool_utils:throw_error("Illegal lib dir for ~p: ~p", + reltool_utils:throw_error("Illegal lib dir for ~w: ~p", [App#app.name, Val]) end; SelectVsn when SelectVsn=:=vsn; SelectVsn=:=lib_dir -> @@ -1642,7 +1642,7 @@ patch_erts_version(RootDir, Apps, Status) -> end; false -> reltool_utils:throw_error( - "erts cannot be found in the root directory ~p", [RootDir]) + "erts cannot be found in the root directory ~tp", [RootDir]) end. libs_to_dirs(RootDir, LibDirs) -> @@ -1669,10 +1669,10 @@ libs_to_dirs(RootDir, LibDirs) -> lists:prefix("erts", F)], app_dirs2(AllLibDirs, [ErtsFiles]); [Duplicate | _] -> - reltool_utils:throw_error("Duplicate library: ~p",[Duplicate]) + reltool_utils:throw_error("Duplicate library: ~tp",[Duplicate]) end; {error, Reason} -> - reltool_utils:throw_error("Missing root library ~p: ~s", + reltool_utils:throw_error("Missing root library ~tp: ~ts", [RootDir,file:format_error(Reason)]) end. @@ -1697,7 +1697,7 @@ app_dirs2([Lib | Libs], Acc) -> Files2 = lists:zf(Filter, Files), app_dirs2(Libs, [Files2 | Acc]); {error, Reason} -> - reltool_utils:throw_error("Illegal library ~p: ~s", + reltool_utils:throw_error("Illegal library ~tp: ~ts", [Lib, file:format_error(Reason)]) end; app_dirs2([], Acc) -> @@ -1756,7 +1756,7 @@ escripts_to_apps([Escript | Escripts], Apps, Status) -> {ok, AF} -> AF; {error, Reason1} -> - reltool_utils:throw_error("Illegal escript ~p: ~p", + reltool_utils:throw_error("Illegal escript ~tp: ~p", [Escript,Reason1]) end, @@ -1838,7 +1838,7 @@ escripts_to_apps([Escript | Escripts], Apps, Status) -> Status2), escripts_to_apps(Escripts, Apps2, Status3); {error, Reason2} -> - reltool_utils:throw_error("Illegal escript ~p: ~p", + reltool_utils:throw_error("Illegal escript ~tp: ~p", [Escript,Reason2]) end; escripts_to_apps([], Apps, Status) -> @@ -1901,7 +1901,7 @@ init_escript_app(AppName, EscriptAppName, Dir, Info, Mods, Apps, Status) -> case lists:keymember(AppName, #app.name, Apps) of true -> reltool_utils:throw_error( - "~p: Application name clash. Escript ~p contains application ~p.", + "~w: Application name clash. Escript ~tp contains application ~tp.", [AppName,Dir,AppName]); false -> {App2, Status} @@ -1986,7 +1986,7 @@ ensure_app_info(#app{is_escript = IsEscript, active_dir = Dir, info = Info}, %% Escript or application which is inlined in an escript {Info, Dir, Status}; ensure_app_info(#app{name = Name, sorted_dirs = []}, _Status) -> - reltool_utils:throw_error("~p: : Missing application directory.",[Name]); + reltool_utils:throw_error("~w: : Missing application directory.",[Name]); ensure_app_info(#app{name = Name, vsn = Vsn, use_selected_vsn = UseSelectedVsn, @@ -2011,8 +2011,8 @@ ensure_app_info(#app{name = Name, Status2; [BadVsn | _] -> reltool_utils:throw_error( - "~p: Application version clash. " - "Multiple directories contains version ~p.", + "~w: Application version clash. " + "Multiple directories contains version ~tp.", [Name,BadVsn]) end, FirstInfo = hd(AllInfo), @@ -2034,8 +2034,8 @@ ensure_app_info(#app{name = Name, {Info, VsnDir, Status3}; false -> reltool_utils:throw_error( - "~p: No application directory contains " - "selected version ~p", [Name,Vsn]) + "~w: No application directory contains " + "selected version ~tp", [Name,Vsn]) end end; true -> diff --git a/lib/reltool/src/reltool_sys_win.erl b/lib/reltool/src/reltool_sys_win.erl index 8e182d02ed..b5d54e6d3e 100644 --- a/lib/reltool/src/reltool_sys_win.erl +++ b/lib/reltool/src/reltool_sys_win.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -181,7 +181,7 @@ do_init([{safe_config, Safe}, {parent, Parent} | Options]) -> end. restart_server_safe_config(true,Parent,Reason) -> - io:format("~p(~p): <ERROR> ~p\n", [?MODULE, ?LINE, Reason]), + io:format("~w(~w): <ERROR> ~p\n", [?MODULE, ?LINE, Reason]), proc_lib:init_ack(Parent, {error,Reason}); restart_server_safe_config(false,Parent,Reason) -> wx:new(), @@ -198,7 +198,7 @@ restart_server_safe_config(false,Parent,Reason) -> ?wxID_OK -> do_init([{safe_config,true},{parent,Parent},?safe_config]); ?wxID_CANCEL -> - io:format("~p(~p): <ERROR> ~p\n", [?MODULE, ?LINE, Reason]), + io:format("~w(~w): <ERROR> ~p\n", [?MODULE, ?LINE, Reason]), proc_lib:init_ack(Parent,{error,Reason}) end. @@ -211,7 +211,7 @@ exit_dialog(Warnings) -> ?wxID_OK -> ok; ?wxID_CANCEL -> - io:format("~p(~p): <ERROR> ~s\n", [?MODULE, ?LINE, Details]), + io:format("~w(~w): <ERROR> ~ts\n", [?MODULE, ?LINE, Details]), exit(Details) end. @@ -249,7 +249,7 @@ loop(S) -> WWs2 = lists:delete(ObjRef, WWs), ?MODULE:loop(S#state{warning_wins = WWs2}); false -> - error_logger:format("~p~p got unexpected " + error_logger:format("~w~w got unexpected " "message:\n\t~p\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) @@ -291,7 +291,7 @@ loop(S) -> S#state.app_wins), ?MODULE:loop(S#state{fgraph_wins = FWs, app_wins = AWs}); Msg -> - error_logger:format("~p~p got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~p\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end. @@ -315,7 +315,7 @@ handle_child_exit({'EXIT', Pid, _Reason} = Exit, FWs, AWs) -> msg_warning({'EXIT', _Pid, shutdown}, Type) when Type =/= unknown -> ok; msg_warning(Exit, Type) -> - error_logger:format("~p~p got unexpected message (~p):\n\t~p\n", + error_logger:format("~w~w got unexpected message (~w):\n\t~p\n", [?MODULE, self(), Type, Exit]). create_window(S) -> @@ -1161,7 +1161,7 @@ handle_system_event(#state{sys = Sys} = S, Sys2 = Sys#sys{incl_cond = AppCond}, do_set_sys(S#state{sys = Sys2}); handle_system_event(S, Event, ObjRef, UserData) -> - error_logger:format("~p~p got unexpected wx sys event to ~p " + error_logger:format("~w~w got unexpected wx sys event to ~p " "with user data: ~p\n\t ~p\n", [?MODULE, self(), ObjRef, UserData, Event]), S. @@ -1177,13 +1177,13 @@ handle_source_event(S, _UserData) -> case wxTreeCtrl:getItemData(ObjRef, Item) of #root_data{dir = _Dir} -> - %% io:format("Root dialog: ~p\n", [Dir]), + %% io:format("Root dialog: ~tp\n", [Dir]), S; #lib_data{dir = _Dir} -> - %% io:format("Lib dialog: ~p\n", [Dir]), + %% io:format("Lib dialog: ~tp\n", [Dir]), S; #escript_data{file = _File} -> - %% io:format("Escript dialog: ~p\n", [File]), + %% io:format("Escript dialog: ~tp\n", [File]), S; #app_data{name = Name} -> do_open_app(S, Name); @@ -1203,7 +1203,7 @@ handle_source_event(S, #escript_data{file = File} -> wx:batch(fun() -> escript_popup(S, File, Tree, Item) end); #app_data{name = Name} -> - io:format("App menu: ~p\n", [Name]), + io:format("App menu: ~tp\n", [Name]), S; undefined -> S @@ -1223,7 +1223,7 @@ handle_app_event(S, Items = reltool_utils:get_items(ListCtrl), handle_app_button(S, Items, Action); handle_app_event(S, Event, ObjRef, UserData) -> - error_logger:format("~p~p got unexpected wx app event to " + error_logger:format("~w~w got unexpected wx app event to " "~p with user data: ~p\n\t ~p\n", [?MODULE, self(), ObjRef, UserData, Event]), S. @@ -1267,7 +1267,7 @@ move_app(S, {_ItemNo, AppBase}, Action) -> blacklist_del -> undefined; _ -> - error_logger:format("~p~p got unexpected app " + error_logger:format("~w~w got unexpected app " "button event: ~p ~p\n", [?MODULE, self(), Action, AppBase]), OldApp#app.incl_cond diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl index 1f4ce7226a..9cda5dabd3 100644 --- a/lib/reltool/src/reltool_target.erl +++ b/lib/reltool/src/reltool_target.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -296,7 +296,7 @@ do_gen_rel(#rel{name = RelName, vsn = RelVsn, rel_apps = RelApps}, {ErtsName, Erts#app.vsn}, [strip_rel_info(App, RelApps) || App <- MergedApps]}; false -> - reltool_utils:throw_error("Mandatory application ~p is " + reltool_utils:throw_error("Mandatory application ~w is " "not included", [ErtsName]) end. @@ -383,8 +383,8 @@ merge_app(RelName, [] -> App#app{app_type = Type2, info = Info#app_info{incl_apps = InclApps}}; BadIncl -> - reltool_utils:throw_error("~p: These applications are " - "used by release ~s but are " + reltool_utils:throw_error("~w: These applications are " + "used by release ~ts but are " "missing as included_applications " "in the app file: ~p", [Name, RelName, BadIncl]) @@ -865,7 +865,7 @@ strip_sys_files(Relocatable, SysFiles, Apps, ExclRegexps) -> case File of "erts" -> reltool_utils:throw_error("This system is not installed. " - "The directory ~s is missing.", + "The directory ~ts is missing.", [Erts#app.label]); _ when File =:= Erts#app.label -> replace_dyn_erl(Relocatable, Spec); @@ -987,7 +987,7 @@ check_sys(Mandatory, SysFiles) -> do_check_sys(Prefix, Specs) -> case lookup_spec(Prefix, Specs) of [] -> - reltool_utils:throw_error("Mandatory system directory ~s " + reltool_utils:throw_error("Mandatory system directory ~ts " "is not included", [Prefix]); _ -> @@ -1008,8 +1008,8 @@ lookup_spec(Prefix, Specs) -> safe_lookup_spec(Prefix, Specs) -> case lookup_spec(Prefix, Specs) of [] -> - %% io:format("lookup fail ~s:\n\t~p\n", [Prefix, Specs]), - reltool_utils:throw_error("Mandatory system file ~s is " + %% io:format("lookup fail ~ts:\n\t~p\n", [Prefix, Specs]), + reltool_utils:throw_error("Mandatory system file ~ts is " "not included", [Prefix]); Match -> Match @@ -1053,7 +1053,7 @@ spec_lib_files(#sys{root_dir = RootDir, check_apps([Mandatory | Names], Apps) -> case lists:keymember(Mandatory, #app.name, Apps) of false -> - reltool_utils:throw_error("Mandatory application ~p is " + reltool_utils:throw_error("Mandatory application ~w is " "not included in ~p", [Mandatory, Apps]); true -> @@ -1144,13 +1144,13 @@ spec_dir(Dir) -> Base, [spec_dir(filename:join([Dir, F])) || F <- Files]}; error -> - reltool_utils:throw_error("list dir ~s failed", [Dir]) + reltool_utils:throw_error("list dir ~ts failed", [Dir]) end; {ok, #file_info{type = regular}} -> %% Plain file {copy_file, Base}; _ -> - reltool_utils:throw_error("read file info ~s failed", [Dir]) + reltool_utils:throw_error("read file info ~ts failed", [Dir]) end. spec_mod(Mod, DebugInfo) -> @@ -1284,7 +1284,7 @@ do_eval_spec({archive, Archive, Options, Files}, {ok, _} -> ok; {error, Reason} -> - reltool_utils:throw_error("create archive ~s failed: ~p", + reltool_utils:throw_error("create archive ~ts failed: ~p", [ArchiveFile, Reason]) end; do_eval_spec({copy_file, File}, _OrigSourceDir, SourceDir, TargetDir) -> @@ -1473,7 +1473,7 @@ do_install(RelName, TargetDir) -> ok = release_handler:create_RELEASES(TargetDir2, RelFile), ok; _ -> - reltool_utils:throw_error("~s: Illegal data file syntax", [DataFile]) + reltool_utils:throw_error("~ts: Illegal data file syntax",[DataFile]) end. nativename(Dir) -> diff --git a/lib/reltool/src/reltool_utils.erl b/lib/reltool/src/reltool_utils.erl index 6149d6ef06..edccb889b1 100644 --- a/lib/reltool/src/reltool_utils.erl +++ b/lib/reltool/src/reltool_utils.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -106,7 +106,7 @@ normalize_dir([], Path) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% prim_consult(Bin) when is_binary(Bin) -> - case erl_scan:string(binary_to_list(Bin)) of + case erl_scan:string(unicode:characters_to_list(Bin,encoding(Bin))) of {ok, Tokens, _EndLine} -> prim_parse(Tokens, []); {error, {_ErrorLine, Module, Reason}, _EndLine} -> @@ -120,6 +120,14 @@ prim_consult(FullName) when is_list(FullName) -> {error, file:format_error(enoent)} end. +encoding(Bin) when is_binary(Bin) -> + case epp:read_encoding_from_binary(Bin) of + none -> + epp:default_encoding(); + E -> + E + end. + prim_parse(Tokens, Acc) -> case lists:splitwith(fun(T) -> element(1,T) =/= dot end, Tokens) of {[], []} -> @@ -423,7 +431,7 @@ scroll_size(ObjRef) -> safe_keysearch(Key, Pos, List, Mod, Line) -> case lists:keysearch(Key, Pos, List) of false -> - io:format("~p(~p): lists:keysearch(~p, ~p, ~p) -> false\n", + io:format("~w(~w): lists:keysearch(~p, ~w, ~p) -> false\n", [Mod, Line, Key, Pos, List]), erlang:error({Mod, Line, lists, keysearch, [Key, Pos, List]}); {value, Val} -> @@ -455,7 +463,7 @@ create_dir(Dir) -> ok; {error, Reason} -> Text = file:format_error(Reason), - throw_error("create dir ~s: ~s", [Dir, Text]) + throw_error("create dir ~ts: ~ts", [Dir, Text]) end. list_dir(Dir) -> @@ -464,7 +472,7 @@ list_dir(Dir) -> Files; error -> Text = file:format_error(enoent), - throw_error("list dir ~s: ~s", [Dir, Text]) + throw_error("list dir ~ts: ~ts", [Dir, Text]) end. read_file_info(File) -> @@ -473,7 +481,7 @@ read_file_info(File) -> Info; {error, Reason} -> Text = file:format_error(Reason), - throw_error("read file info ~s: ~s", [File, Text]) + throw_error("read file info ~ts: ~ts", [File, Text]) end. write_file_info(File, Info) -> @@ -482,7 +490,7 @@ write_file_info(File, Info) -> ok; {error, Reason} -> Text = file:format_error(Reason), - throw_error("write file info ~s: ~s", [File, Text]) + throw_error("write file info ~ts: ~ts", [File, Text]) end. read_file(File) -> @@ -491,7 +499,7 @@ read_file(File) -> Bin; {error, Reason} -> Text = file:format_error(Reason), - throw_error("read file ~s: ~s", [File, Text]) + throw_error("read file ~ts: ~ts", [File, Text]) end. write_file(File, IoList) -> @@ -500,7 +508,7 @@ write_file(File, IoList) -> ok; {error, Reason} -> Text = file:format_error(Reason), - throw_error("write file ~s: ~s", [File, Text]) + throw_error("write file ~ts: ~ts", [File, Text]) end. recursive_delete(Dir) -> @@ -516,7 +524,7 @@ recursive_delete(Dir) -> ok; {error, Reason} -> Text = file:format_error(Reason), - throw_error("delete file ~s: ~s\n", [Dir, Text]) + throw_error("delete file ~ts: ~ts\n", [Dir, Text]) end; false -> delete(Dir, regular) @@ -530,7 +538,7 @@ delete(File, Type) -> ok; {error, Reason} -> Text = file:format_error(Reason), - throw_error("delete file ~s: ~s\n", [File, Text]) + throw_error("delete file ~ts: ~ts\n", [File, Text]) end. do_delete(File, regular) -> @@ -569,11 +577,11 @@ copy_file(From, To) -> ok; {error, Reason} -> Text = file:format_error(Reason), - throw_error("copy file ~s -> ~s: ~s\n", [From, To, Text]) + throw_error("copy file ~ts -> ~ts: ~ts\n", [From, To, Text]) end; error -> Text = file:format_error(enoent), - throw_error("copy file ~s -> ~s: ~s\n", [From, To, Text]) + throw_error("copy file ~ts -> ~ts: ~ts\n", [From, To, Text]) end. throw_error(Format, Args) -> @@ -586,13 +594,10 @@ decode_regexps(Key, {add, Regexps}, Old) when is_list(Regexps) -> decode_regexps(_Key, {del, Regexps}, Old) when is_list(Regexps) -> [Re || Re <- Old, not lists:member(Re#regexp.source, Regexps)]; decode_regexps(Key, Regexps, _Old) when is_list(Regexps) -> - do_decode_regexps(Key, Regexps, []); -decode_regexps(Key, Regexps, _Old) when is_list(Regexps) -> - Text = lists:flatten(io_lib:format("~p", [{Key, Regexps}])), - throw({error, "Illegal option: " ++ Text}). + do_decode_regexps(Key, Regexps, []). do_decode_regexps(Key, [Regexp | Regexps], Acc) -> - case catch re:compile(Regexp, []) of + case catch re:compile(Regexp, [unicode]) of {ok, MP} -> do_decode_regexps(Key, Regexps, diff --git a/lib/reltool/test/reltool_server_SUITE.erl b/lib/reltool/test/reltool_server_SUITE.erl index 8d71865508..23338d9ecd 100644 --- a/lib/reltool/test/reltool_server_SUITE.erl +++ b/lib/reltool/test/reltool_server_SUITE.erl @@ -1,7 +1,8 @@ +%% -*- coding: utf-8 -*- %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -42,6 +43,11 @@ end_per_suite(Config) -> reltool_test_lib:end_per_suite(Config). init_per_testcase(Func,Config) -> + Node = full_node_name(?NODE_NAME), + case net_adm:ping(Node) of + pong -> stop_node(Node); + pang -> ok + end, reltool_test_lib:init_per_testcase(Func,Config). end_per_testcase(Func,Config) -> reltool_test_lib:end_per_testcase(Func,Config). @@ -60,6 +66,7 @@ all() -> create_script, create_script_sort, create_target, + create_target_unicode, create_embedded, create_standalone, create_standalone_beam, @@ -750,6 +757,72 @@ create_target(_Config) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Generate target system + +create_target_unicode(Config) -> + DataDir = ?config(data_dir,Config), + + %% If file name translation mode is unicode, then use unicode + %% characters release name (which will be used as file name for + %% .rel, .script and .boot) + RelNamePrefix = + case file:native_name_encoding() of + utf8 -> + "Unicode test αβ"; + latin1 -> + "Unicode test" + end, + + %% Configure the server + RelName1 = RelNamePrefix, + RelName2 = RelNamePrefix ++ " with SASL", + RelVsn = "1.0", + Sys = + {sys, + [ + {root_dir, code:root_dir()}, + {lib_dirs, [filename:join(DataDir,"unicode")]}, + {app_file, all}, + {incl_cond,exclude}, + {boot_rel, RelName2}, + {rel, RelName1, RelVsn, [stdlib, kernel, ua]}, + {rel, RelName2, RelVsn, [sasl, stdlib, kernel, ua]}, + {app, kernel, [{incl_cond, include}]}, + {app, stdlib, [{incl_cond, include}]}, + {app, sasl, [{incl_cond, include}]}, + {app, ua, [{incl_cond, include}]} + ]}, + + %% Generate target file + TargetDir = filename:join([?WORK_DIR, "target_unicode"]), + ?m(ok, reltool_utils:recursive_delete(TargetDir)), + ?m(ok, file:make_dir(TargetDir)), + ?log("SPEC: ~p\n", [reltool:get_target_spec([{config, Sys}])]), + ok = ?m(ok, reltool:create_target([{config, Sys}], TargetDir)), + + %% Start a node + Erl = filename:join([TargetDir, "bin", "erl"]), + {ok, Node} = ?msym({ok, _}, start_node(?NODE_NAME, Erl)), + + + %% The ua application has a unicode string as description - check + %% that it is translated correctly. + wait_for_app(Node,ua,50), + Apps = rpc:call(Node,application,which_applications,[]), + ?m({ua,"Application for testing unicode in reltool - αβ","1.0"}, + lists:keyfind(ua,1,Apps)), + + %% Check that the release name is correct (really only + %% insteresting if file name translation mode is utf8) + [{RelName,_,_,_}] = + ?msym([{_,_,_,_}],rpc:call(Node,release_handler,which_releases,[])), + ?m(true,lists:prefix(RelNamePrefix,RelName)), + + ?msym(ok, stop_node(Node)), + + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Generate embedded target system create_embedded(_Config) -> @@ -810,11 +883,11 @@ create_standalone(_Config) -> ?msym(ok, stop_node(Node)), %% Execute escript - Expected = iolist_to_binary(["Root dir: ", RootDir, "\n" - "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", - "Smp: false\n", - "ExitCode:0"]), - io:format("Expected: ~s\n", [Expected]), + Expected = s2b(["Root dir: ", RootDir, "\n" + "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", + "Emuarg: [\"emuvalue\"]\n", + "ExitCode:0"]), + io:format("Expected: ~ts\n", [Expected]), ?m(Expected, run(BinDir, EscriptName, "-arg1 arg2 arg3")), ok. @@ -857,10 +930,11 @@ create_standalone_beam(Config) -> ?msym(ok, stop_node(Node)), %% Execute escript - Expected = iolist_to_binary(["Root dir: ", RootDir, "\n" - "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", - "ExitCode:0"]), - io:format("Expected: ~s\n", [Expected]), + Expected = s2b(["Module: mymod\n" + "Root dir: ", RootDir, "\n" + "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", + "ExitCode:0"]), + io:format("Expected: ~ts\n", [Expected]), ?m(Expected, run(BinDir, EscriptName, "-arg1 arg2 arg3")), ok. @@ -909,10 +983,11 @@ create_standalone_app(Config) -> ?msym(ok, stop_node(Node)), %% Execute escript - Expected = iolist_to_binary(["Root dir: ", RootDir, "\n" - "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", - "ExitCode:0"]), - io:format("Expected: ~s\n", [Expected]), + Expected = s2b(["Module: mymod\n" + "Root dir: ", RootDir, "\n" + "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", + "ExitCode:0"]), + io:format("Expected: ~ts\n", [Expected]), ?m(Expected, run(BinDir, EscriptName, "-arg1 arg2 arg3")), ok. @@ -995,19 +1070,20 @@ create_multiple_standalone(Config) -> ?msym(ok, stop_node(Node)), %% Execute escript1 - Expected1 = iolist_to_binary(["Root dir: ", RootDir, "\n" - "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", - "Smp: false\n", - "ExitCode:0"]), - io:format("Expected1: ~s\n", [Expected1]), + Expected1 = s2b(["Root dir: ", RootDir, "\n" + "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", + "Emuarg: [\"emuvalue\"]\n", + "ExitCode:0"]), + io:format("Expected1: ~ts\n", [Expected1]), ?m(Expected1, run(BinDir, EscriptName1, "-arg1 arg2 arg3")), %% Execute escript2 - Expected2 = iolist_to_binary(["Root dir: ", RootDir, "\n" - "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", - "ExitCode:0"]), - io:format("Expected2: ~s\n", [Expected2]), + Expected2 = s2b(["Module: mymod\n" + "Root dir: ", RootDir, "\n" + "Script args: [\"-arg1\",\"arg2\",\"arg3\"]\n", + "ExitCode:0"]), + io:format("Expected2: ~ts\n", [Expected2]), ?m(Expected2, run(BinDir, EscriptName2, "-arg1 arg2 arg3")), ok. @@ -1094,6 +1170,7 @@ create_slim(Config) -> "-sasl", "releases_dir", EscapedQuote++TargetRelDir++EscapedQuote], {ok, Node} = ?msym({ok, _}, start_node(?NODE_NAME, Erl, Args)), ?msym(RootDir, rpc:call(Node, code, root_dir, [])), + wait_for_app(Node,sasl,50), ?msym([{RelName,RelVsn,_,permanent}], rpc:call(Node,release_handler,which_releases,[])), ?msym(ok, stop_node(Node)), @@ -2420,7 +2497,7 @@ start_node(Name, ErlPath) -> start_node(Name, ErlPath, Args0) -> FullName = full_node_name(Name), Args = mk_node_args(Name, Args0), - io:format("Starting node ~p: ~s~n", + io:format("Starting node ~p: ~ts~n", [FullName, lists:flatten([[X," "] || X <- [ErlPath|Args]])]), %io:format("open_port({spawn_executable, ~p}, [{args,~p}])~n",[ErlPath,Args]), case open_port({spawn_executable, ErlPath}, [{args,Args}]) of @@ -2436,9 +2513,19 @@ start_node(Name, ErlPath, Args0) -> end. stop_node(Node) -> - monitor_node(Node, true), - spawn(Node, fun () -> halt() end), - receive {nodedown, Node} -> ok end. + rpc:call(Node,erlang,halt,[]), + wait_for_node_down(Node,50). + +wait_for_node_down(Node,0) -> + test_server:fail({cant_terminate_node,Node}); +wait_for_node_down(Node,N) -> + case net_adm:ping(Node) of + pong -> + timer:sleep(1000), + wait_for_node_down(Node,N-1); + pang -> + ok + end. mk_node_args(Name, Args) -> Pa = filename:dirname(code:which(?MODULE)), @@ -2485,6 +2572,22 @@ wait_for_process(Node, Name, N) when is_integer(N), N > 0 -> ok end. +wait_for_app(_Node, Name, 0) -> + {error, Name}; +wait_for_app(Node, Name, N) when is_integer(N), N > 0 -> + case rpc:call(Node,application,which_applications,[]) of + {badrpc,Reason} -> + test_server:fail({failed_to_get_applications,Reason}); + Apps -> + case lists:member(Name,Apps) of + false -> + timer:sleep(1000), + wait_for_app(Node, Name, N-1); + true -> + ok + end + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Run escript @@ -2513,7 +2616,7 @@ do_run(Dir, Cmd) -> Res = get_data(Port, []), receive {Port,{exit_status,ExitCode}} -> - iolist_to_binary([Res,"ExitCode:"++integer_to_list(ExitCode)]) + s2b([Res,"ExitCode:"++integer_to_list(ExitCode)]) end. get_data(Port, SoFar) -> @@ -2537,3 +2640,9 @@ expected_output([], _) -> []; expected_output(Bin, _) when is_binary(Bin) -> Bin. + +%% Convert the given list to a binary with the same encoding as the +%% file name translation mode +s2b(List) -> + Enc = file:native_name_encoding(), + unicode:characters_to_binary(List,Enc,Enc). diff --git a/lib/reltool/test/reltool_server_SUITE_data/escript/someapp-1.0/src/mymod.erl b/lib/reltool/test/reltool_server_SUITE_data/escript/someapp-1.0/src/mymod.erl index b6c71c666d..c315f926d3 100644 --- a/lib/reltool/test/reltool_server_SUITE_data/escript/someapp-1.0/src/mymod.erl +++ b/lib/reltool/test/reltool_server_SUITE_data/escript/someapp-1.0/src/mymod.erl @@ -22,5 +22,6 @@ %%%----------------------------------------------------------------- %%% escript main function main(Args) -> + io:format("Module: ~w\n", [?MODULE]), io:format("Root dir: ~s\n", [code:root_dir()]), io:format("Script args: ~p\n", [Args]). diff --git a/lib/reltool/test/reltool_server_SUITE_data/unicode/ua-1.0/ebin/ua.app b/lib/reltool/test/reltool_server_SUITE_data/unicode/ua-1.0/ebin/ua.app new file mode 100644 index 0000000000..6f571b7179 --- /dev/null +++ b/lib/reltool/test/reltool_server_SUITE_data/unicode/ua-1.0/ebin/ua.app @@ -0,0 +1,7 @@ +% -*- coding: utf-8 -*- +{application, ua, + [{description, "Application for testing unicode in reltool - αβ"}, + {vsn, "1.0"}, + {modules,[]}, + {registered, []}, + {applications, [kernel, stdlib]}]}. diff --git a/lib/reltool/vsn.mk b/lib/reltool/vsn.mk index b1edf770aa..9df2fc8406 100644 --- a/lib/reltool/vsn.mk +++ b/lib/reltool/vsn.mk @@ -1 +1 @@ -RELTOOL_VSN = 0.6.2 +RELTOOL_VSN = 0.6.3 diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 35cefb4b78..cd59be1e63 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/sasl/doc/src/notes.xml b/lib/sasl/doc/src/notes.xml index 1d6c45e64a..f54b04c223 100644 --- a/lib/sasl/doc/src/notes.xml +++ b/lib/sasl/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,22 @@ </header> <p>This document describes the changes made to the SASL application.</p> +<section><title>SASL 2.3.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Some updates are made to systools and release_handler for + handling of unicode.</p> + <p> + Own Id: OTP-10782</p> + </item> + </list> + </section> + +</section> + <section><title>SASL 2.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl index 1ff3eb96eb..c1b715b970 100644 --- a/lib/sasl/src/release_handler.erl +++ b/lib/sasl/src/release_handler.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -41,7 +41,8 @@ %% Internal exports, a client release_handler may call this functions. -export([do_write_release/3, do_copy_file/2, do_copy_files/2, do_copy_files/1, do_rename_files/1, do_remove_files/1, - remove_file/1, do_write_file/2, do_ensure_RELEASES/1]). + remove_file/1, do_write_file/2, do_write_file/3, + do_ensure_RELEASES/1]). -record(state, {unpurged = [], root, @@ -254,7 +255,7 @@ check_timeout(_Else) -> false. new_emulator_upgrade(Vsn, Opts) -> Result = call({install_release, Vsn, reboot, Opts}), error_logger:info_msg( - "~p:install_release(~p,~p) completed after node restart " + "~w:install_release(~p,~p) completed after node restart " "with new emulator version~nResult: ~p~n",[?MODULE,Vsn,Opts,Result]), Result. @@ -1128,7 +1129,7 @@ new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn,RelDir,Masters) -> {ok,[FC]} -> FC; {error,Error1} -> - io:format("Warning: ~p can not read ~p: ~p~n", + io:format("Warning: ~w can not read ~p: ~p~n", [?MODULE,FromFile,Error1]), [] end, @@ -1138,7 +1139,7 @@ new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn,RelDir,Masters) -> {ok,[ToConfig]} -> [lists:keyfind(App,1,ToConfig) || App <- [kernel,stdlib,sasl]]; {error,Error2} -> - io:format("Warning: ~p can not read ~p: ~p~n", + io:format("Warning: ~w can not read ~p: ~p~n", [?MODULE,ToFile,Error2]), [false,false,false] end, @@ -1597,7 +1598,9 @@ remove_file(File) -> end. do_write_file(File, Str) -> - case file:open(File, [write]) of + do_write_file(File, Str, []). +do_write_file(File, Str, FileOpts) -> + case file:open(File, [write | FileOpts]) of {ok, Fd} -> io:put_chars(Fd, Str), file:close(Fd), diff --git a/lib/sasl/src/release_handler_1.erl b/lib/sasl/src/release_handler_1.erl index 93d12cf609..b37ae2f944 100644 --- a/lib/sasl/src/release_handler_1.erl +++ b/lib/sasl/src/release_handler_1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -624,7 +624,7 @@ get_proc_state(Proc) -> maybe_supervisor_which_children(suspended, Name, Pid) -> error_logger:error_msg("release_handler: a which_children call" - " to ~p (~p) was avoided. This supervisor" + " to ~p (~w) was avoided. This supervisor" " is suspended and should likely be upgraded" " differently. Exiting ...~n", [Name, Pid]), error(suspended_supervisor); @@ -635,7 +635,7 @@ maybe_supervisor_which_children(State, Name, Pid) -> Res; Other -> error_logger:error_msg("release_handler: ~p~nerror during" - " a which_children call to ~p (~p)." + " a which_children call to ~p (~w)." " [State: ~p] Exiting ... ~n", [Other, Name, Pid, State]), error(which_children_failed) @@ -647,7 +647,7 @@ maybe_get_dynamic_mods(Name, Pid) -> Res; Other -> error_logger:error_msg("release_handler: ~p~nerror during a" - " get_modules call to ~p (~p)," + " get_modules call to ~p (~w)," " there may be an error in it's" " childspec. Exiting ...~n", [Other, Name, Pid]), diff --git a/lib/sasl/src/si_sasl_supp.erl b/lib/sasl/src/si_sasl_supp.erl index 97f906a5f8..c4fc0c5f08 100644 --- a/lib/sasl/src/si_sasl_supp.erl +++ b/lib/sasl/src/si_sasl_supp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/sasl/src/systools_lib.erl b/lib/sasl/src/systools_lib.erl index 1b6ea125d9..6618baa2aa 100644 --- a/lib/sasl/src/systools_lib.erl +++ b/lib/sasl/src/systools_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -42,7 +42,11 @@ file_term2binary(FileIn, FileOut) -> %%______________________________________________________________________ %% read_term(File) -> {ok, Term} | Error - +%% +%% This is really an own implementation of file:consult/1, except it +%% returns one term and not a list of terms. Keeping the function +%% instead of using file:consult - for backwards compatibility with +%% error reasons. read_term(File) -> case file:open(File, [read]) of {ok, Stream} -> @@ -54,6 +58,7 @@ read_term(File) -> end. read_term_from_stream(Stream, File) -> + _ = epp:set_encoding(Stream), R = io:request(Stream, {get_until,'',erl_scan,tokens,[1]}), case R of {ok,Toks,_EndLine} -> @@ -176,11 +181,11 @@ add_dirs(RegName, Dirs, Root) -> regexp_match(RegName, D0, Root) -> case file:list_dir(D0) of {ok, Files} when length(Files) > 0 -> - case re:compile(RegName) of + case re:compile(RegName,[unicode]) of {ok, MP} -> FR = fun(F) -> - case re:run(F, MP) of - {match,[{0,N}]} when N == length(F) -> + case re:run(F, MP, [{capture,first,list}]) of + {match,[F]} -> % All of F matches DirF = join(D0, F, Root), case dir_p(DirF) of true -> diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index 9b2e2c809b..193dbb64bf 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -156,10 +156,10 @@ return(ok,Warnings,Flags) -> _ -> case member(warnings_as_errors,Flags) of true -> - io:format("~s",[format_warning(Warnings, true)]), + io:format("~ts",[format_warning(Warnings, true)]), error; false -> - io:format("~s",[format_warning(Warnings)]), + io:format("~ts",[format_warning(Warnings)]), ok end end; @@ -168,7 +168,7 @@ return({error,Mod,Error},_,Flags) -> true -> {error,Mod,Error}; _ -> - io:format("~s",[Mod:format_error(Error)]), + io:format("~ts",[Mod:format_error(Error)]), error end. @@ -1970,17 +1970,11 @@ is_app_type(_) -> false. % check if a term is a string. -string_p([H|T]) when is_integer(H), H >= $ , H < 255 -> - string_p(T); -string_p([$\n|T]) -> string_p(T); -string_p([$\r|T]) -> string_p(T); -string_p([$\t|T]) -> string_p(T); -string_p([$\v|T]) -> string_p(T); -string_p([$\b|T]) -> string_p(T); -string_p([$\f|T]) -> string_p(T); -string_p([$\e|T]) -> string_p(T); -string_p([]) -> true; -string_p(_) -> false. +string_p(S) -> + case unicode:characters_to_list(S) of + S -> true; + _ -> false + end. % check if a term is a list of two tuples with the first % element as an atom. @@ -2203,31 +2197,31 @@ format_error({illegal_applications,Names}) -> io_lib:format("Illegal applications in the release file: ~p~n", [Names]); format_error({missing_mandatory_app,Name}) -> - io_lib:format("Mandatory application ~p must be specified in the release file~n", + io_lib:format("Mandatory application ~w must be specified in the release file~n", [Name]); format_error({mandatory_app,Name,Type}) -> - io_lib:format("Mandatory application ~p must be of type 'permanent' in the release file. Is '~p'.~n", + io_lib:format("Mandatory application ~w must be of type 'permanent' in the release file. Is '~p'.~n", [Name,Type]); format_error({duplicate_register,Dups}) -> - io_lib:format("Duplicated register names: ~n~s", + io_lib:format("Duplicated register names: ~n~ts", [map(fun({{Reg,App1,_,_},{Reg,App2,_,_}}) -> - io_lib:format("\t~p registered in ~p and ~p~n", + io_lib:format("\t~w registered in ~w and ~w~n", [Reg,App1,App2]) end, Dups)]); format_error({undefined_applications,Apps}) -> io_lib:format("Undefined applications: ~p~n",[Apps]); format_error({duplicate_modules,Dups}) -> - io_lib:format("Duplicated modules: ~n~s", + io_lib:format("Duplicated modules: ~n~ts", [map(fun({{Mod,_,App1,_,_},{Mod,_,App2,_,_}}) -> - io_lib:format("\t~p specified in ~p and ~p~n", + io_lib:format("\t~w specified in ~w and ~w~n", [Mod,App1,App2]) end, Dups)]); format_error({included_and_used, Dups}) -> io_lib:format("Applications both used and included: ~p~n",[Dups]); format_error({duplicate_include, Dups}) -> - io_lib:format("Duplicated application included: ~n~s", + io_lib:format("Duplicated application included: ~n~ts", [map(fun({{Name,App1,_,_},{Name,App2,_,_}}) -> - io_lib:format("\t~p included in ~p and ~p~n", + io_lib:format("\t~w included in ~w and ~w~n", [Name,App1,App2]) end, Dups)]); format_error({modules,ModErrs}) -> @@ -2238,11 +2232,11 @@ format_error({not_found,File}) -> io_lib:format("File not found: ~p~n",[File]); format_error({parse,File,{Line,Mod,What}}) -> Str = Mod:format_error(What), - io_lib:format("~s:~p: ~s\n",[File, Line, Str]); + io_lib:format("~ts:~w: ~ts\n",[File, Line, Str]); format_error({read,File}) -> io_lib:format("Cannot read ~p~n",[File]); format_error({open,File,Error}) -> - io_lib:format("Cannot open ~p - ~s~n", + io_lib:format("Cannot open ~p - ~ts~n", [File,file:format_error(Error)]); format_error({tar_error,What}) -> form_tar_err(What); @@ -2258,24 +2252,21 @@ format_errors(ListOfErrors) -> form_err({bad_application_name,{Name,Found}}) -> io_lib:format("~p: Mismatched application id: ~p~n",[Name,Found]); form_err({error_reading, {Name, What}}) -> - io_lib:format("~p: ~s~n",[Name,form_reading(What)]); + io_lib:format("~p: ~ts~n",[Name,form_reading(What)]); form_err({module_not_found,App,Mod}) -> - io_lib:format("~p: Module (~p) not found~n",[App,Mod]); -form_err({{vsn_diff,File},{Mod,Vsn,App,_,_}}) -> - io_lib:format("~p: Module (~p) version (~p) differs in file ~p~n", - [App,Mod,Vsn,File]); + io_lib:format("~w: Module (~w) not found~n",[App,Mod]); form_err({error_add_appl, {Name, {tar_error, What}}}) -> - io_lib:format("~p: ~s~n",[Name,form_tar_err(What)]); + io_lib:format("~p: ~ts~n",[Name,form_tar_err(What)]); form_err(E) -> io_lib:format("~p~n",[E]). form_reading({not_found,File}) -> io_lib:format("File not found: ~p~n",[File]); form_reading({application_vsn, {Name,Vsn}}) -> - io_lib:format("Application ~s with version ~p not found~n",[Name, Vsn]); + io_lib:format("Application ~ts with version ~p not found~n",[Name, Vsn]); form_reading({parse,File,{Line,Mod,What}}) -> Str = Mod:format_error(What), - io_lib:format("~s:~p: ~s\n",[File, Line, Str]); + io_lib:format("~ts:~w: ~ts\n",[File, Line, Str]); form_reading({read,File}) -> io_lib:format("Cannot read ~p~n",[File]); form_reading({{bad_param, P},_}) -> @@ -2291,15 +2282,15 @@ form_reading({no_valid_version, {{_, SVsn}, {_, File, FVsn}}}) -> io_lib:format("No valid version (~p) of .app file found. Found file ~p with version ~p~n", [SVsn, File, FVsn]); form_reading({parse_error, {File, Line, Error}}) -> - io_lib:format("Parse error in file: ~p. Line: ~p Error: ~p; ~n", [File, Line, Error]); + io_lib:format("Parse error in file: ~p. Line: ~w Error: ~p; ~n", [File, Line, Error]); form_reading(W) -> io_lib:format("~p~n",[W]). form_tar_err({open, File, Error}) -> - io_lib:format("Cannot open tar file ~s - ~p~n", + io_lib:format("Cannot open tar file ~ts - ~ts~n", [File, erl_tar:format_error(Error)]); form_tar_err({add, File, Error}) -> - io_lib:format("Cannot add file ~s to tar file - ~s~n", + io_lib:format("Cannot add file ~ts to tar file - ~ts~n", [File, erl_tar:format_error(Error)]). %% Format warning @@ -2317,23 +2308,23 @@ format_warning(Warnings, Werror) -> map(fun({warning,W}) -> form_warn(Prefix, W) end, Warnings). form_warn(Prefix, {source_not_found,{Mod,_,App,_,_}}) -> - io_lib:format("~s~p: Source code not found: ~p.erl~n", + io_lib:format("~ts~w: Source code not found: ~w.erl~n", [Prefix,App,Mod]); form_warn(Prefix, {{parse_error, File},{_,_,App,_,_}}) -> - io_lib:format("~s~p: Parse error: ~p~n", + io_lib:format("~ts~w: Parse error: ~p~n", [Prefix,App,File]); form_warn(Prefix, {obj_out_of_date,{Mod,_,App,_,_}}) -> - io_lib:format("~s~p: Object code (~p) out of date~n", + io_lib:format("~ts~w: Object code (~w) out of date~n", [Prefix,App,Mod]); form_warn(Prefix, {exref_undef, Undef}) -> F = fun({M,F,A}) -> - io_lib:format("~sUndefined function ~p:~p/~p~n", + io_lib:format("~tsUndefined function ~w:~w/~w~n", [Prefix,M,F,A]) end, map(F, Undef); form_warn(Prefix, missing_sasl) -> - io_lib:format("~s: Missing application sasl. " + io_lib:format("~ts: Missing application sasl. " "Can not upgrade with this release~n", [Prefix]); form_warn(Prefix, What) -> - io_lib:format("~s ~p~n", [Prefix,What]). + io_lib:format("~ts ~p~n", [Prefix,What]). diff --git a/lib/sasl/src/systools_rc.erl b/lib/sasl/src/systools_rc.erl index cf5cca7cb3..54c327410d 100644 --- a/lib/sasl/src/systools_rc.erl +++ b/lib/sasl/src/systools_rc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -904,7 +904,7 @@ format_error({bad_op_before_point_of_no_return, Instruction}) -> io_lib:format("Bad instruction ~p~nbefore point_of_no_return~n", [Instruction]); format_error({no_object_code, Mod}) -> - io_lib:format("No load_object_code found for module: ~p~n", [Mod]); + io_lib:format("No load_object_code found for module: ~w~n", [Mod]); format_error({suspended_not_resumed, Mods}) -> io_lib:format("Suspended but not resumed: ~p~n", [Mods]); format_error({resumed_not_suspended, Mods}) -> @@ -916,19 +916,19 @@ format_error({start_not_stop, Mods}) -> format_error({stop_not_start, Mods}) -> io_lib:format("Stopped but not started: ~p~n", [Mods]); format_error({no_such_application, App}) -> - io_lib:format("Started undefined application: ~p~n", [App]); + io_lib:format("Started undefined application: ~w~n", [App]); format_error({removed_application_present, App}) -> - io_lib:format("Removed application present: ~p~n", [App]); + io_lib:format("Removed application present: ~w~n", [App]); format_error(dup_mnesia_backup) -> io_lib:format("Duplicate mnesia_backup~n", []); format_error(bad_mnesia_backup) -> io_lib:format("mnesia_backup in bad position~n", []); format_error({conflicting_versions, Lib, V1, V2}) -> - io_lib:format("Conflicting versions for ~p, ~p and ~p~n", [Lib, V1, V2]); + io_lib:format("Conflicting versions for ~w, ~ts and ~ts~n", [Lib, V1, V2]); format_error({no_appl_vsn, Appl}) -> - io_lib:format("No version specified for application: ~p~n", [Appl]); + io_lib:format("No version specified for application: ~w~n", [Appl]); format_error({no_such_module, Mod}) -> - io_lib:format("No such module: ~p~n", [Mod]); + io_lib:format("No such module: ~w~n", [Mod]); format_error(too_many_point_of_no_return) -> io_lib:format("Too many point_of_no_return~n", []); diff --git a/lib/sasl/src/systools_relup.erl b/lib/sasl/src/systools_relup.erl index 7048184426..716dc2b5ff 100644 --- a/lib/sasl/src/systools_relup.erl +++ b/lib/sasl/src/systools_relup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -494,21 +494,18 @@ get_script_from_appup(Mode, TopApp, BaseVsn, Ws, RUs) -> throw({error, ?MODULE, {no_relup, FName, TopApp, BaseVsn}}) end. -appup_search_for_version(BaseVsn, VsnRUs) -> - appup_search_for_version(BaseVsn, length(BaseVsn), VsnRUs). - -appup_search_for_version(BaseVsn,_,[{BaseVsn,RU}|_]) -> +appup_search_for_version(BaseVsn,[{BaseVsn,RU}|_]) -> {ok,RU}; -appup_search_for_version(BaseVsn,Size,[{Vsn,RU}|VsnRUs]) when is_binary(Vsn) -> - case re:run(BaseVsn,Vsn,[unicode,{capture,first,index}]) of - {match,[{0,Size}]} -> +appup_search_for_version(BaseVsn,[{Vsn,RU}|VsnRUs]) when is_binary(Vsn) -> + case re:run(BaseVsn,Vsn,[unicode,{capture,first,list}]) of + {match,[BaseVsn]} -> {ok, RU}; _ -> - appup_search_for_version(BaseVsn,Size,VsnRUs) + appup_search_for_version(BaseVsn,VsnRUs) end; -appup_search_for_version(BaseVsn,Size,[_|VsnRUs]) -> - appup_search_for_version(BaseVsn,Size,VsnRUs); -appup_search_for_version(_,_,[]) -> +appup_search_for_version(BaseVsn,[_|VsnRUs]) -> + appup_search_for_version(BaseVsn,VsnRUs); +appup_search_for_version(_,[]) -> error. @@ -603,14 +600,15 @@ print_error(Other) -> format_error({file_problem, {"relup", _Posix}}) -> io_lib:format("Could not open file relup~n", []); format_error({file_problem, {File, What}}) -> - io_lib:format("Could not ~p file ~p~n", [get_reason(What), File]); + io_lib:format("Could not ~w file ~ts~n", [get_reason(What), File]); format_error({no_relup, File, App, Vsn}) -> - io_lib:format("No release upgrade script entry for ~p-~s to ~p-~s " - "in file ~p~n", + io_lib:format("No release upgrade script entry for ~w-~ts to ~w-~ts " + "in file ~ts~n", [App#application.name, App#application.vsn, App#application.name, Vsn, File]); format_error({missing_sasl,Release}) -> - io_lib:format("No sasl application in release ~p, ~p. Can not be upgraded.", + io_lib:format("No sasl application in release ~ts, ~ts. " + "Can not be upgraded.", [Release#release.name, Release#release.vsn]); format_error(Error) -> io:format("~p~n", [Error]). @@ -629,16 +627,16 @@ print_warning(W, Opts) -> "*WARNING* " end, S = format_warning(Prefix, W), - io:format("~s", [S]). + io:format("~ts", [S]). format_warning(W) -> format_warning("*WARNING* ", W). format_warning(Prefix, {erts_vsn_changed, {Rel1, Rel2}}) -> - io_lib:format("~sThe ERTS version changed between ~p and ~p~n", + io_lib:format("~tsThe ERTS version changed between ~p and ~p~n", [Prefix, Rel1, Rel2]); format_warning(Prefix, What) -> - io_lib:format("~s~p~n",[Prefix, What]). + io_lib:format("~ts~p~n",[Prefix, What]). get_reason({error, {open, _, _}}) -> open; diff --git a/lib/sasl/test/systools_SUITE.erl b/lib/sasl/test/systools_SUITE.erl index 878d582e6b..367cab1d77 100644 --- a/lib/sasl/test/systools_SUITE.erl +++ b/lib/sasl/test/systools_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012. All Rights Reserved. +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -40,7 +40,8 @@ -export([all/0,suite/0,groups/0,init_per_group/2,end_per_group/2]). --export([script_options/1, normal_script/1, no_mod_vsn_script/1, +-export([script_options/1, normal_script/1, unicode_script/1, + unicode_script/2, no_mod_vsn_script/1, wildcard_script/1, variable_script/1, no_sasl_script/1, abnormal_script/1, src_tests_script/1, crazy_script/1, included_script/1, included_override_script/1, @@ -75,7 +76,7 @@ all() -> groups() -> [{script, [], - [script_options, normal_script, no_mod_vsn_script, + [script_options, normal_script, unicode_script, no_mod_vsn_script, wildcard_script, variable_script, abnormal_script, no_sasl_script, src_tests_script, crazy_script, included_script, included_override_script, @@ -250,6 +251,62 @@ normal_script(Config) when is_list(Config) -> ok. +%% make_script: Test make_script with unicode .app file +unicode_script(Config) when is_list(Config) -> + UnicodeStr = [945,946], % alhpa beta in greek letters + + {LatestDir, LatestName} = create_script({unicode,UnicodeStr},Config), + + DataDir = filename:absname(?copydir), + UnicodeApp = fname([DataDir, "d_unicode", "lib", "ua-1.0"]), + TarFile = fname(?privdir, "unicode_app.tgz"), + {ok, Tar} = erl_tar:open(TarFile, [write, compressed]), + ok = erl_tar:add(Tar, UnicodeApp, "ua-1.0", [compressed]), + ok = erl_tar:close(Tar), + + UnicodeLibDir = fname([DataDir, "d_unicode", UnicodeStr]), + P1 = fname([UnicodeLibDir, "ua-1.0", "ebin"]), + + %% Need to do this on a separate node to make sure it has unicode + %% filename mode (+fnu*) + {ok,HostStr} = inet:gethostname(), + Host = list_to_atom(HostStr), + {ok,Node} = ct_slave:start(Host,unicode_script_node,[{erl_flags,"+fnui"}]), + + ok = rpc:call(Node,erl_tar,extract, + [TarFile, [{cwd,UnicodeLibDir},compressed]]), + + true = rpc:call(Node,code,add_patha,[P1]), + + ok = rpc:call(Node,file,set_cwd,[LatestDir]), + + ok = rpc:call(Node,systools,make_script,[filename:basename(LatestName), + [local]]), + + {ok, Script} = rpc:call(Node,file,consult,[LatestName++".script"]), + + %% For debug purpose - print script to log + io:format("~tp~n",[Script]), + + %% check that script contains unicode strings in + %% 1. release version (set in ?MODULE:do_create_script) + [{script,{"Test release",UnicodeStr},Instr}] = Script, + + %% 2. application description (set in ua.app in data dir) + [AppInfo] = [X || {apply,{application,load,[{application,ua,X}]}} <- Instr], + {description,UnicodeStr} = lists:keyfind(description,1,AppInfo), + + %% 3. path (directory name where unicode_app.tgz is extracted) + true = lists:member({path,[P1]},Instr), + + ok. + +unicode_script(cleanup,Config) -> + _ = ct_slave:stop(unicode_script_node), + file:delete(fname(?privdir, "unicode_app.tgz")), + ok. + + %% make_script: %% Modules specified without version in .app file (db-3.1). %% Note that this is now the normal way - i.e. systools now ignores @@ -2090,15 +2147,20 @@ create_script(current_all_future_erts,Config) -> do_create_script(current_all_future_erts,Config,"99.99",Apps); create_script(current_all_future_sasl,Config) -> Apps = [{kernel,current},{stdlib,current},{sasl,"9.9"},{db,"2.1"},{fe,"3.1"}], - do_create_script(current_all_future_sasl,Config,current,Apps). + do_create_script(current_all_future_sasl,Config,current,Apps); +create_script({unicode,RelVsn},Config) -> + Apps = core_apps(current) ++ [{ua,"1.0"}], + do_create_script(unicode,RelVsn,Config,current,Apps). do_create_script(Id,Config,ErtsVsn,AppVsns) -> + do_create_script(Id,string:to_upper(atom_to_list(Id)),Config,ErtsVsn,AppVsns). +do_create_script(Id,RelVsn,Config,ErtsVsn,AppVsns) -> PrivDir = ?privdir, Name = fname(PrivDir, Id), {ok,Fd} = file:open(Name++".rel",write), RelfileContent = - {release,{"Test release", string:to_upper(atom_to_list(Id))}, + {release,{"Test release", RelVsn}, {erts,erts_vsn(ErtsVsn)}, app_vsns(AppVsns)}, io:format(Fd,"~p.~n",[RelfileContent]), diff --git a/lib/sasl/test/systools_SUITE_data/d_unicode/lib/ua-1.0/ebin/ua.app b/lib/sasl/test/systools_SUITE_data/d_unicode/lib/ua-1.0/ebin/ua.app new file mode 100644 index 0000000000..3d38a3dde4 --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_unicode/lib/ua-1.0/ebin/ua.app @@ -0,0 +1,9 @@ +%% -*- coding: utf-8 -*- +{application, ua, + [{description, "αβ"}, + {vsn, "1.0"}, + {modules, [ua1]}, + {registered, []}, + {applications, []}, + {env, []}, + {start, {ua1, start, []}}]}. diff --git a/lib/sasl/test/systools_SUITE_data/d_unicode/lib/ua-1.0/src/ua1.erl b/lib/sasl/test/systools_SUITE_data/d_unicode/lib/ua-1.0/src/ua1.erl new file mode 100644 index 0000000000..e988e80f3d --- /dev/null +++ b/lib/sasl/test/systools_SUITE_data/d_unicode/lib/ua-1.0/src/ua1.erl @@ -0,0 +1,2 @@ +-module(ua1). +-vsn("1.0"). diff --git a/lib/sasl/vsn.mk b/lib/sasl/vsn.mk index 93ba34d448..0e52133166 100644 --- a/lib/sasl/vsn.mk +++ b/lib/sasl/vsn.mk @@ -1 +1 @@ -SASL_VSN = 2.3 +SASL_VSN = 2.3.1 diff --git a/lib/snmp/src/compile/snmpc.erl b/lib/snmp/src/compile/snmpc.erl index d94810bc0a..2f065dddac 100644 --- a/lib/snmp/src/compile/snmpc.erl +++ b/lib/snmp/src/compile/snmpc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 3c289cdc46..12175d9a29 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -29,6 +29,29 @@ <file>notes.xml</file> </header> +<section><title>Ssh 2.1.4</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Better quality on the error messages for when key + exchange failed.</p> + <p> + Own Id: OTP-10553 Aux Id: seq12152 </p> + </item> + <item> + <p> + Fix link to documentation for ssh:connect/3,4. Thanks to + Martin H�ssler.</p> + <p> + Own Id: OTP-10862</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 2.1.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index f57ee13460..bd0d3d49dd 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -91,7 +91,8 @@ </type> <desc> <p>Connects to an SSH server. No channel is started. This is done - by calling ssh_connect:session_channel/2.</p> + by calling + <seealso marker="ssh_connection#session_channel/2">ssh_connection:session_channel/[2, 4]</seealso>.</p> <p>Options are:</p> <taglist> <tag><c><![CDATA[{user_dir, string()}]]></c></tag> diff --git a/lib/ssh/src/ssh.appup.src b/lib/ssh/src/ssh.appup.src index c4b5aa256b..f4986410ab 100644 --- a/lib/ssh/src/ssh.appup.src +++ b/lib/ssh/src/ssh.appup.src @@ -19,6 +19,7 @@ {"%VSN%", [ + {<<"2.1.4">>, [{load_module, ssh_sftp, soft_purge, soft_purge, []}]}, {<<"2.1.3">>, [{restart_application, ssh}]}, {<<"2.1.2">>, [{restart_application, ssh}]}, {<<"2.1.1">>, [{restart_application, ssh}]}, @@ -27,7 +28,7 @@ {<<"1\\.*">>, [{restart_application, ssh}]} ], [ - {<<"2.1.3">>, [{restart_application, ssh}]}, + {<<"2.1.4">>, [{load_module, ssh_sftp, soft_purge, soft_purge, []}]}, {<<"2.1.2">>, [{restart_application, ssh}]}, {<<"2.1.1">>, [{restart_application, ssh}]}, {<<"2.1">>,[{restart_application, ssh}]}, diff --git a/lib/ssh/src/ssh_connection_manager.erl b/lib/ssh/src/ssh_connection_manager.erl index 9536eb9dec..79a11c4b20 100644 --- a/lib/ssh/src/ssh_connection_manager.erl +++ b/lib/ssh/src/ssh_connection_manager.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index f3afbe01bf..10167a9223 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -403,7 +403,7 @@ init([Cm, ChannelId, Timeout]) -> rep_buf = <<>>, inf = new_inf()}}; failure -> - {stop, {error, "server failed to start sftp subsystem"}}; + {stop, "server failed to start sftp subsystem"}; Error -> {stop, Error} end. diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index a47a55b707..98d59d01de 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2012. All Rights Reserved. +%% Copyright Ericsson AB 2004-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index efcb11f88f..dceec52464 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -137,16 +137,16 @@ end_per_testcase(_Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -app_test(doc) -> - ["Application consistency test."]; +app_test() -> + [{doc, "App lication consistency test."}]. app_test(Config) when is_list(Config) -> ?t:app_test(ssh), ok. %%-------------------------------------------------------------------- -misc_ssh_options(doc) -> - ["Test that we can set some misc options not tested elsewhere, " - "some options not yet present are not decided if we should support or " - "if they need thier own test case."]; +misc_ssh_options() -> + [{doc, "Test that we can set some misc options not tested elsewhere, " + "some options not yet present are not decided if we should support or " + "if they need thier own test case."}]. misc_ssh_options(Config) when is_list(Config) -> SystemDir = filename:join(?config(priv_dir, Config), system), UserDir = ?config(priv_dir, Config), @@ -163,8 +163,8 @@ misc_ssh_options(Config) when is_list(Config) -> basic_test([{client_opts, CMiscOpt1 ++ ClientOpts}, {server_opts, SMiscOpt1 ++ ServerOpts}]). %%-------------------------------------------------------------------- -exec(doc) -> - ["Test api function ssh_connection:exec"]; +exec() -> + [{doc, "Test api function ssh_connection:exec"}]. exec(Config) when is_list(Config) -> process_flag(trap_exit, true), SystemDir = filename:join(?config(priv_dir, Config), system), @@ -205,8 +205,8 @@ exec(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -exec_compressed(doc) -> - ["Test that compression option works"]; +exec_compressed() -> + [{doc, "Test that compression option works"}]. exec_compressed(Config) when is_list(Config) -> process_flag(trap_exit, true), SystemDir = filename:join(?config(priv_dir, Config), system), @@ -234,8 +234,8 @@ exec_compressed(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -idle_time(doc) -> - ["Idle timeout test"]; +idle_time() -> + [{doc, "Idle timeout test"}]. idle_time(Config) -> SystemDir = filename:join(?config(priv_dir, Config), system), UserDir = ?config(priv_dir, Config), @@ -256,8 +256,8 @@ idle_time(Config) -> end, ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -rekey(doc) -> - ["Idle timeout test"]; +rekey() -> + [{doc, "Idle timeout test"}]. rekey(Config) -> SystemDir = filename:join(?config(priv_dir, Config), system), UserDir = ?config(priv_dir, Config), @@ -278,8 +278,8 @@ rekey(Config) -> ssh:stop_daemon(Pid) end. %%-------------------------------------------------------------------- -shell(doc) -> - ["Test that ssh:shell/2 works"]; +shell() -> + [{doc, "Test that ssh:shell/2 works"}]. shell(Config) when is_list(Config) -> process_flag(trap_exit, true), SystemDir = filename:join(?config(priv_dir, Config), system), @@ -300,9 +300,9 @@ shell(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -daemon_already_started(doc) -> - ["Test that get correct error message if you try to start a daemon", - "on an adress that already runs a daemon see also seq10667" ]; +daemon_already_started() -> + [{doc, "Test that get correct error message if you try to start a daemon", + "on an adress that already runs a daemon see also seq10667"}]. daemon_already_started(Config) when is_list(Config) -> SystemDir = ?config(data_dir, Config), UserDir = ?config(priv_dir, Config), @@ -317,8 +317,8 @@ daemon_already_started(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -server_password_option(doc) -> - ["validate to server that uses the 'password' option"]; +server_password_option() -> + [{doc, "validate to server that uses the 'password' option"}]. server_password_option(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth @@ -351,8 +351,8 @@ server_password_option(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -server_userpassword_option(doc) -> - ["validate to server that uses the 'password' option"]; +server_userpassword_option() -> + [{doc, "validate to server that uses the 'password' option"}]. server_userpassword_option(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth @@ -387,8 +387,8 @@ server_userpassword_option(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -known_hosts(doc) -> - ["check that known_hosts is updated correctly"]; +known_hosts() -> + [{doc, "check that known_hosts is updated correctly"}]. known_hosts(Config) when is_list(Config) -> SystemDir = ?config(data_dir, Config), PrivDir = ?config(priv_dir, Config), @@ -414,8 +414,8 @@ known_hosts(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -pass_phrase(doc) -> - ["Test that we can use keyes protected by pass phrases"]; +pass_phrase() -> + [{doc, "Test that we can use keyes protected by pass phrases"}]. pass_phrase(Config) when is_list(Config) -> process_flag(trap_exit, true), SystemDir = filename:join(?config(priv_dir, Config), system), @@ -435,8 +435,8 @@ pass_phrase(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -internal_error(doc) -> - ["Test that client does not hang if disconnects due to internal error"]; +internal_error() -> + [{doc,"Test that client does not hang if disconnects due to internal error"}]. internal_error(Config) when is_list(Config) -> process_flag(trap_exit, true), SystemDir = filename:join(?config(priv_dir, Config), system), @@ -452,8 +452,8 @@ internal_error(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -send(doc) -> - ["Test ssh_connection:send/3"]; +send() -> + [{doc, "Test ssh_connection:send/3"}]. send(Config) when is_list(Config) -> process_flag(trap_exit, true), SystemDir = filename:join(?config(priv_dir, Config), system), @@ -473,8 +473,8 @@ send(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -close(doc) -> - ["Simulate that we try to close an already closed connection"]; +close() -> + [{doc, "Simulate that we try to close an already closed connection"}]. close(Config) when is_list(Config) -> SystemDir = ?config(data_dir, Config), PrivDir = ?config(priv_dir, Config), diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index acaf3d6eeb..6c781e0e91 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -82,8 +82,8 @@ end_per_testcase(_Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -simple_exec(doc) -> - ["Simple openssh connectivity test for ssh_connection:exec"]; +simple_exec() -> + [{doc, "Simple openssh connectivity test for ssh_connection:exec"}]. simple_exec(Config) when is_list(Config) -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, @@ -113,8 +113,8 @@ simple_exec(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -small_cat(doc) -> - ["Use 'cat' to echo small data block back to us."]; +small_cat() -> + [{doc, "Use 'cat' to echo small data block back to us."}]. small_cat(Config) when is_list(Config) -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, @@ -148,8 +148,8 @@ small_cat(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -big_cat(doc) -> - ["Use 'cat' to echo large data block back to us."]; +big_cat() -> + [{doc,"Use 'cat' to echo large data block back to us."}]. big_cat(Config) when is_list(Config) -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, @@ -197,8 +197,8 @@ big_cat(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -send_after_exit(doc) -> - ["Send channel data after the channel has been closed."]; +send_after_exit() -> + [{doc, "Send channel data after the channel has been closed."}]. send_after_exit(Config) when is_list(Config) -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, @@ -236,8 +236,8 @@ send_after_exit(Config) when is_list(Config) -> ok end. %%-------------------------------------------------------------------- -interrupted_send(doc) -> - ["Use a subsystem that echos n char and then sends eof to cause a channel exit partway through a large send."]; +interrupted_send() -> + [{doc, "Use a subsystem that echos n char and then sends eof to cause a channel exit partway through a large send."}]. interrupted_send(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), diff --git a/lib/ssh/test/ssh_echo_server.erl b/lib/ssh/test/ssh_echo_server.erl index 007b00c373..315ffecfd7 100644 --- a/lib/ssh/test/ssh_echo_server.erl +++ b/lib/ssh/test/ssh_echo_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2012. All Rights Reserved. +%% Copyright Ericsson AB 2005-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,7 +21,7 @@ %%% Description: Example ssh server -module(ssh_echo_server). --behaviour(ssh_subsytem). +-behaviour(ssh_daemon_channel). -record(state, { n, id, diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index 232161d029..56b1363b7a 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2012. All Rights Reserved. +%% Copyright Ericsson AB 2005-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -41,7 +41,9 @@ suite() -> all() -> [{group, erlang_server}, - {group, openssh_server}]. + {group, openssh_server}, + sftp_nonexistent_subsystem + ]. init_per_suite(Config) -> @@ -76,9 +78,7 @@ init_per_group(erlang_server, Config) -> ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, PrivDir}, {user_passwords, - [{?USER, ?PASSWD}]}, - {failfun, - fun ssh_test_lib:failfun/2}]), + [{?USER, ?PASSWD}]}]), [{group, erlang_server}, {sftpd, Sftpd} | Config]; init_per_group(openssh_server, Config) -> @@ -100,6 +100,17 @@ end_per_group(_, Config) -> %%-------------------------------------------------------------------- +init_per_testcase(sftp_nonexistent_subsystem, Config) -> + PrivDir = ?config(priv_dir, Config), + SysDir = ?config(data_dir, Config), + Sftpd = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, PrivDir}, + {subsystems, []}, + {user_passwords, + [{?USER, ?PASSWD}]} + ]), + [{sftpd, Sftpd} | Config]; + init_per_testcase(Case, Config) -> prep(Config), TmpConfig0 = lists:keydelete(watchdog, 1, Config), @@ -129,6 +140,8 @@ init_per_testcase(Case, Config) -> [{sftp, Sftp}, {watchdog, Dog} | TmpConfig] end. +end_per_testcase(sftp_nonexistent_subsystem, Config) -> + Config; end_per_testcase(rename_file, Config) -> PrivDir = ?config(priv_dir, Config), NewFileName = filename:join(PrivDir, "test.txt"), @@ -145,8 +158,8 @@ end_per_testcase(Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -open_close_file(doc) -> - ["Test API functions open/3 and close/2"]; +open_close_file() -> + [{doc, "Test API functions open/3 and close/2"}]. open_close_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "sftp.txt"), @@ -165,8 +178,8 @@ open_close_file(Server, File, Mode) -> ok = ssh_sftp:close(Server, Handle). %%-------------------------------------------------------------------- -open_close_dir(doc) -> - ["Test API functions opendir/2 and close/2"]; +open_close_dir() -> + [{doc, "Test API functions opendir/2 and close/2"}]. open_close_dir(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), {Sftp, _} = ?config(sftp, Config), @@ -177,8 +190,8 @@ open_close_dir(Config) when is_list(Config) -> {error, _} = ssh_sftp:opendir(Sftp, FileName). %%-------------------------------------------------------------------- -read_file(doc) -> - ["Test API funtion read_file/2"]; +read_file() -> + [{doc, "Test API funtion read_file/2"}]. read_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "sftp.txt"), @@ -187,8 +200,8 @@ read_file(Config) when is_list(Config) -> {ok, Data} = file:read_file(FileName). %%-------------------------------------------------------------------- -read_dir(doc) -> - ["Test API function list_dir/2"]; +read_dir() -> + [{doc,"Test API function list_dir/2"}]. read_dir(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), {Sftp, _} = ?config(sftp, Config), @@ -196,8 +209,8 @@ read_dir(Config) when is_list(Config) -> ct:pal("sftp list dir: ~p~n", [Files]). %%-------------------------------------------------------------------- -write_file(doc) -> - ["Test API function write_file/2"]; +write_file() -> + [{doc, "Test API function write_file/2"}]. write_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "sftp.txt"), @@ -208,8 +221,8 @@ write_file(Config) when is_list(Config) -> {ok, Data} = file:read_file(FileName). %%-------------------------------------------------------------------- -remove_file(doc) -> - ["Test API function delete/2"]; +remove_file() -> + [{doc,"Test API function delete/2"}]. remove_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "sftp.txt"), @@ -222,8 +235,8 @@ remove_file(Config) when is_list(Config) -> false = lists:member(filename:basename(FileName), NewFiles), {error, _} = ssh_sftp:delete(Sftp, FileName). %%-------------------------------------------------------------------- -rename_file(doc) -> - ["Test API function rename_file/2"]; +rename_file() -> + [{doc, "Test API function rename_file/2"}]. rename_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "sftp.txt"), @@ -242,8 +255,8 @@ rename_file(Config) when is_list(Config) -> true = lists:member(filename:basename(NewFileName), NewFiles). %%-------------------------------------------------------------------- -mk_rm_dir(doc) -> - ["Test API functions make_dir/2, del_dir/2"]; +mk_rm_dir() -> + [{doc,"Test API functions make_dir/2, del_dir/2"}]. mk_rm_dir(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), {Sftp, _} = ?config(sftp, Config), @@ -256,8 +269,8 @@ mk_rm_dir(Config) when is_list(Config) -> {error, _} = ssh_sftp:del_dir(Sftp, PrivDir). %%-------------------------------------------------------------------- -links(doc) -> - ["Tests API function make_symlink/3"]; +links() -> + [{doc,"Tests API function make_symlink/3"}]. links(Config) when is_list(Config) -> case os:type() of {win32, _} -> @@ -273,8 +286,8 @@ links(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -retrieve_attributes(doc) -> - ["Test API function read_file_info/3"]; +retrieve_attributes() -> + [{doc, "Test API function read_file_info/3"}]. retrieve_attributes(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "sftp.txt"), @@ -287,8 +300,8 @@ retrieve_attributes(Config) when is_list(Config) -> ct:pal("SFTP: ~p FILE: ~p~n", [FileInfo, NewFileInfo]). %%-------------------------------------------------------------------- -set_attributes(doc) -> - ["Test API function write_file_info/3"]; +set_attributes() -> + [{doc,"Test API function write_file_info/3"}]. set_attributes(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -303,8 +316,8 @@ set_attributes(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -async_read(doc) -> - ["Test API aread/3"]; +async_read() -> + [{doc,"Test API aread/3"}]. async_read(Config) when is_list(Config) -> {Sftp, _} = ?config(sftp, Config), PrivDir = ?config(priv_dir, Config), @@ -321,8 +334,8 @@ async_read(Config) when is_list(Config) -> ct:fail(Msg) end. %%-------------------------------------------------------------------- -async_write(doc) -> - ["Test API awrite/3"]; +async_write() -> + [{doc,"Test API awrite/3"}]. async_write(Config) when is_list(Config) -> {Sftp, _} = ?config(sftp, Config), PrivDir = ?config(priv_dir, Config), @@ -340,8 +353,8 @@ async_write(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -position(doc) -> - ["Test API functions position/3"]; +position() -> + [{doc, "Test API functions position/3"}]. position(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -370,8 +383,8 @@ position(Config) when is_list(Config) -> {ok, "2"} = ssh_sftp:read(Sftp, Handle, 1). %%-------------------------------------------------------------------- -pos_read(doc) -> - ["Test API functions pread/3 and apread/3"]; +pos_read() -> + [{doc,"Test API functions pread/3 and apread/3"}]. pos_read(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -396,8 +409,8 @@ pos_read(Config) when is_list(Config) -> {ok, NewData1} = ssh_sftp:pread(Sftp, Handle, {bof, 4}, 4). %%-------------------------------------------------------------------- -pos_write(doc) -> - ["Test API functions pwrite/4 and apwrite/4"]; +pos_write() -> + [{doc,"Test API functions pwrite/4 and apwrite/4"}]. pos_write(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -423,6 +436,17 @@ pos_write(Config) when is_list(Config) -> {ok, NewData1} = ssh_sftp:read_file(Sftp, FileName). %%-------------------------------------------------------------------- +sftp_nonexistent_subsystem() -> + [{doc, "Try to execute sftp subsystem on a server that does not support it"}]. +sftp_nonexistent_subsystem(Config) when is_list(Config) -> + {_,Host, Port} = ?config(sftpd, Config), + {error,"server failed to start sftp subsystem"} = + ssh_sftp:start_channel(Host, Port, + [{user_interaction, false}, + {user, ?USER}, {password, ?PASSWD}, + {silently_accept_hosts, true}]). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- prep(Config) -> diff --git a/lib/ssh/test/ssh_sftpd_SUITE.erl b/lib/ssh/test/ssh_sftpd_SUITE.erl index 5aa46872ee..7b22e45d5e 100644 --- a/lib/ssh/test/ssh_sftpd_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -163,8 +163,8 @@ end_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -open_close_file(doc) -> - ["Test SSH_FXP_OPEN and SSH_FXP_CLOSE commands"]; +open_close_file() -> + [{doc, "Test SSH_FXP_OPEN and SSH_FXP_CLOSE commands"}]. open_close_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -194,8 +194,8 @@ open_close_file(Config) when is_list(Config) -> ?SSH_FXF_OPEN_EXISTING). %%-------------------------------------------------------------------- -open_close_dir(doc) -> - ["Test SSH_FXP_OPENDIR and SSH_FXP_CLOSE commands"]; +open_close_dir() -> + [{doc,"Test SSH_FXP_OPENDIR and SSH_FXP_CLOSE commands"}]. open_close_dir(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), {Cm, Channel} = ?config(sftp, Config), @@ -221,8 +221,8 @@ open_close_dir(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -read_file(doc) -> - ["Test SSH_FXP_READ command"]; +read_file() -> + [{doc, "Test SSH_FXP_READ command"}]. read_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -244,8 +244,8 @@ read_file(Config) when is_list(Config) -> {ok, Data} = file:read_file(FileName). %%-------------------------------------------------------------------- -read_dir(doc) -> - ["Test SSH_FXP_READDIR command"]; +read_dir() -> + [{doc,"Test SSH_FXP_READDIR command"}]. read_dir(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), {Cm, Channel} = ?config(sftp, Config), @@ -255,8 +255,8 @@ read_dir(Config) when is_list(Config) -> ok = read_dir(Handle, Cm, Channel, ReqId). %%-------------------------------------------------------------------- -write_file(doc) -> - ["Test SSH_FXP_WRITE command"]; +write_file() -> + [{doc, "Test SSH_FXP_WRITE command"}]. write_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -279,8 +279,8 @@ write_file(Config) when is_list(Config) -> {ok, Data} = file:read_file(FileName). %%-------------------------------------------------------------------- -remove_file(doc) -> - ["Test SSH_FXP_REMOVE command"]; +remove_file() -> + [{doc, "Test SSH_FXP_REMOVE command"}]. remove_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -300,8 +300,8 @@ remove_file(Config) when is_list(Config) -> remove(PrivDir, Cm, Channel, NewReqId). %%-------------------------------------------------------------------- -rename_file(doc) -> - ["Test SSH_FXP_RENAME command"]; +rename_file() -> + [{doc, "Test SSH_FXP_RENAME command"}]. rename_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -337,8 +337,8 @@ rename_file(Config) when is_list(Config) -> ?SSH_FXP_RENAME_ATOMIC). %%-------------------------------------------------------------------- -mk_rm_dir(doc) -> - ["Test SSH_FXP_MKDIR and SSH_FXP_RMDIR command"]; +mk_rm_dir() -> + [{doc, "Test SSH_FXP_MKDIR and SSH_FXP_RMDIR command"}]. mk_rm_dir(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), {Cm, Channel} = ?config(sftp, Config), @@ -360,8 +360,8 @@ mk_rm_dir(Config) when is_list(Config) -> _/binary>>, _} = rmdir(DirName, Cm, Channel, NewReqId2). %%-------------------------------------------------------------------- -real_path(doc) -> - ["Test SSH_FXP_REALPATH command"]; +real_path() -> + [{doc, "Test SSH_FXP_REALPATH command"}]. real_path(Config) when is_list(Config) -> case os:type() of {win32, _} -> @@ -388,8 +388,6 @@ real_path(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -links(doc) -> - []; links(Config) when is_list(Config) -> case os:type() of {win32, _} -> @@ -417,8 +415,8 @@ links(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -retrieve_attributes(doc) -> - ["Test SSH_FXP_STAT, SSH_FXP_LSTAT AND SSH_FXP_FSTAT commands"]; +retrieve_attributes() -> + [{"Test SSH_FXP_STAT, SSH_FXP_LSTAT AND SSH_FXP_FSTAT commands"}]. retrieve_attributes(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -482,8 +480,8 @@ retrieve_attributes(Config) when is_list(Config) -> end, AttrValues). %%-------------------------------------------------------------------- -set_attributes(doc) -> - ["Test SSH_FXP_SETSTAT AND SSH_FXP_FSETSTAT commands"]; +set_attributes() -> + [{doc, "Test SSH_FXP_SETSTAT AND SSH_FXP_FSETSTAT commands"}]. set_attributes(Config) when is_list(Config) -> case os:type() of {win32, _} -> @@ -540,8 +538,8 @@ set_attributes(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -ver3_rename(doc) -> - ["Test that ver3 rename message is handled OTP 6352"]; +ver3_rename() -> + [{doc, "Test that ver3 rename message is handled OTP 6352"}]. ver3_rename(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -554,8 +552,8 @@ ver3_rename(Config) when is_list(Config) -> rename(FileName, NewFileName, Cm, Channel, ReqId, 3, 0). %%-------------------------------------------------------------------- -relpath(doc) -> - ["Check that realpath works ok seq10670"]; +relpath() -> + [{doc, "Check that realpath works ok seq10670"}]. relpath(Config) when is_list(Config) -> ReqId = 0, {Cm, Channel} = ?config(sftp, Config), @@ -577,8 +575,8 @@ relpath(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -sshd_read_file(doc) -> - ["Test SSH_FXP_READ command, using sshd-server"]; +sshd_read_file() -> + [{doc,"Test SSH_FXP_READ command, using sshd-server"}]. sshd_read_file(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), FileName = filename:join(PrivDir, "test.txt"), @@ -598,8 +596,9 @@ sshd_read_file(Config) when is_list(Config) -> read_file(Handle, 100, 0, Cm, Channel, NewReqId), {ok, Data} = file:read_file(FileName). -ver6_basic(doc) -> - ["Test SFTP Version 6"]; +%%-------------------------------------------------------------------- +ver6_basic() -> + [{doc, "Test SFTP Version 6"}]. ver6_basic(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), %FileName = filename:join(PrivDir, "test.txt"), diff --git a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl index 8f722941d4..cc34cc0793 100644 --- a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2012. All Rights Reserved. +%% Copyright Ericsson AB 2007-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -145,9 +145,9 @@ end_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- %% Test cases starts here. ------------------------------------------- %%-------------------------------------------------------------------- -close_file(doc) -> - ["Test that sftpd closes its fildescriptors after compleating the " - "transfer OTP-6350"]; +close_file() -> + [{doc, "Test that sftpd closes its fildescriptors after compleating the " + "transfer OTP-6350"}]. close_file(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -165,10 +165,10 @@ close_file(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -quit(doc) -> - [" When the sftp client ends the session the " +quit() -> + [{doc, " When the sftp client ends the session the " "server will now behave correctly and not leave the " - "client hanging. OTP-6349"]; + "client hanging. OTP-6349"}]. quit(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -197,9 +197,9 @@ quit(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -file_cb(doc) -> - ["Test that it is possible to change the callback module for" - " the sftpds filehandling. OTP-6356"]; +file_cb() -> + [{"Test that it is possible to change the callback module for" + " the sftpds filehandling. OTP-6356"}]. file_cb(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -245,8 +245,6 @@ file_cb(Config) when is_list(Config) -> alt_file_handler_check(alt_del_dir). %%-------------------------------------------------------------------- -root_dir(doc) -> - [""]; root_dir(Config) when is_list(Config) -> {Sftp, _} = ?config(sftp, Config), FileName = "test.txt", @@ -258,16 +256,15 @@ root_dir(Config) when is_list(Config) -> ct:pal("Listing: ~p~n", [Listing]). %%-------------------------------------------------------------------- -list_dir_limited(doc) -> - [""]; list_dir_limited(Config) when is_list(Config) -> {Sftp, _} = ?config(sftp, Config), {ok, Listing} = ssh_sftp:list_dir(Sftp, "."), ct:pal("Listing: ~p~n", [Listing]). -ver6_basic(doc) -> - ["Test some version 6 features"]; +%%-------------------------------------------------------------------- +ver6_basic() -> + [{doc, "Test some version 6 features"}]. ver6_basic(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), NewDir = filename:join(PrivDir, "testdir2"), diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index 99dc76e12d..8b5343cecc 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,7 +49,9 @@ groups() -> erlang_client_openssh_server_setenv, erlang_client_openssh_server_publickey_rsa, erlang_client_openssh_server_publickey_dsa, - erlang_client_openssh_server_password]}, + erlang_client_openssh_server_password, + erlang_client_openssh_server_nonexistent_subsystem + ]}, {erlang_server, [], [erlang_server_openssh_client_exec, erlang_server_openssh_client_exec_compressed, erlang_server_openssh_client_pulic_key_dsa]} @@ -99,8 +101,8 @@ end_per_testcase(_TestCase, _Config) -> %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -erlang_shell_client_openssh_server(doc) -> - ["Test that ssh:shell/2 works"]; +erlang_shell_client_openssh_server() -> + [{doc, "Test that ssh:shell/2 works"}]. erlang_shell_client_openssh_server(Config) when is_list(Config) -> process_flag(trap_exit, true), @@ -126,8 +128,8 @@ erlang_shell_client_openssh_server(Config) when is_list(Config) -> end. %-------------------------------------------------------------------- -erlang_client_openssh_server_exec(doc) -> - ["Test api function ssh_connection:exec"]; +erlang_client_openssh_server_exec() -> + [{doc, "Test api function ssh_connection:exec"}]. erlang_client_openssh_server_exec(Config) when is_list(Config) -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, @@ -165,8 +167,8 @@ erlang_client_openssh_server_exec(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -erlang_client_openssh_server_exec_compressed(doc) -> - ["Test that compression option works"]; +erlang_client_openssh_server_exec_compressed() -> + [{doc, "Test that compression option works"}]. erlang_client_openssh_server_exec_compressed(Config) when is_list(Config) -> ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, @@ -188,8 +190,8 @@ erlang_client_openssh_server_exec_compressed(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- -erlang_server_openssh_client_exec(doc) -> - ["Test that exec command works."]; +erlang_server_openssh_client_exec() -> + [{doc, "Test that exec command works."}]. erlang_server_openssh_client_exec(Config) when is_list(Config) -> SystemDir = ?config(data_dir, Config), @@ -219,8 +221,8 @@ erlang_server_openssh_client_exec(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -erlang_server_openssh_client_exec_compressed(doc) -> - ["Test that exec command works."]; +erlang_server_openssh_client_exec_compressed() -> + [{doc, "Test that exec command works."}]. erlang_server_openssh_client_exec_compressed(Config) when is_list(Config) -> SystemDir = ?config(data_dir, Config), @@ -247,8 +249,8 @@ erlang_server_openssh_client_exec_compressed(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -erlang_client_openssh_server_setenv(doc) -> - ["Test api function ssh_connection:setenv"]; +erlang_client_openssh_server_setenv() -> + [{doc, "Test api function ssh_connection:setenv"}]. erlang_client_openssh_server_setenv(Config) when is_list(Config) -> ConnectionRef = @@ -290,8 +292,8 @@ erlang_client_openssh_server_setenv(Config) when is_list(Config) -> %% setenv not meaningfull on erlang ssh daemon! %%-------------------------------------------------------------------- -erlang_client_openssh_server_publickey_rsa(doc) -> - ["Validate using rsa publickey."]; +erlang_client_openssh_server_publickey_rsa() -> + [{doc, "Validate using rsa publickey."}]. erlang_client_openssh_server_publickey_rsa(Config) when is_list(Config) -> {ok,[[Home]]} = init:get_argument(home), KeyFile = filename:join(Home, ".ssh/id_rsa"), @@ -317,8 +319,8 @@ erlang_client_openssh_server_publickey_rsa(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -erlang_client_openssh_server_publickey_dsa(doc) -> - ["Validate using dsa publickey."]; +erlang_client_openssh_server_publickey_dsa() -> + [{doc, "Validate using dsa publickey."}]. erlang_client_openssh_server_publickey_dsa(Config) when is_list(Config) -> {ok,[[Home]]} = init:get_argument(home), KeyFile = filename:join(Home, ".ssh/id_dsa"), @@ -342,8 +344,8 @@ erlang_client_openssh_server_publickey_dsa(Config) when is_list(Config) -> {skip, "no ~/.ssh/id_dsa"} end. %%-------------------------------------------------------------------- -erlang_server_openssh_client_pulic_key_dsa(doc) -> - ["Validate using dsa publickey."]; +erlang_server_openssh_client_pulic_key_dsa() -> + [{doc, "Validate using dsa publickey."}]. erlang_server_openssh_client_pulic_key_dsa(Config) when is_list(Config) -> SystemDir = ?config(data_dir, Config), PrivDir = ?config(priv_dir, Config), @@ -369,8 +371,8 @@ erlang_server_openssh_client_pulic_key_dsa(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- -erlang_client_openssh_server_password(doc) -> - ["Test client password option"]; +erlang_client_openssh_server_password() -> + [{doc, "Test client password option"}]. erlang_client_openssh_server_password(Config) when is_list(Config) -> %% to make sure we don't public-key-auth UserDir = ?config(data_dir, Config), @@ -402,6 +404,20 @@ erlang_client_openssh_server_password(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- + +erlang_client_openssh_server_nonexistent_subsystem() -> + [{doc, "Test client password option"}]. +erlang_client_openssh_server_nonexistent_subsystem(Config) when is_list(Config) -> + + ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, + [{user_interaction, false}, + silently_accept_hosts]), + + {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), + + failure = ssh_connection:subsystem(ConnectionRef, ChannelId, "foo", infinity). + +%%-------------------------------------------------------------------- % %% Not possible to send password with openssh without user interaction %% diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 9fc4b0522e..8f6aac1f5d 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 2.1.4 +SSH_VSN = 2.1.5 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 299850333d..a61f52b809 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -26,7 +26,35 @@ </header> <p>This document describes the changes made to the SSL application.</p> - <section><title>SSL 5.2</title> + <section><title>SSL 5.2.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Transport callback handling is changed so that gen_tcp is + treated as a special case where inet will be called + directly for functions such as setopts, as gen_tcp does + not have its own setopts. This will enable users to use + the transport callback for other customizations such as + websockets.</p> + <p> + Own Id: OTP-10847</p> + </item> + <item> + <p> + Follow up to OTP-10451 solved in ssl-5.2 R16A. Make sure + format_error return good strings. Replace confusing + legacy atoms with more descriptive atoms.</p> + <p> + Own Id: OTP-10864</p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 5.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index ab468c8d6b..6979fb5b5e 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -278,11 +278,18 @@ fun(OtpCert :: #'OTPCertificate'{}, Event :: {bad_cert, Reason :: atom()} | <p> {bad_cert, cert_expired}, {bad_cert, invalid_issuer}, {bad_cert, invalid_signature}, {bad_cert, unknown_ca},{bad_cert, selfsigned_peer}, {bad_cert, name_not_permitted}, {bad_cert, missing_basic_constraint}, {bad_cert, invalid_key_usage}</p> </item> + <tag>{versions, [protocol()]}</tag> + <item>TLS protocol versions that will be supported by started clients and servers. + This option overrides the application environment option <c>protocol_version</c>. If the + environment option is not set it defaults to all versions supported by the SSL application. See also + <seealso marker="ssl:ssl_app">ssl(6)</seealso> + </item> + <tag>{hibernate_after, integer()|undefined}</tag> - <item>When an integer-value is specified, the <code>ssl_connection</code> + <item>When an integer-value is specified, the <c>ssl_connection</c> will go into hibernation after the specified number of milliseconds of inactivity, thus reducing its memory footprint. When - <code>undefined</code> is specified (this is the default), the process + <c>undefined</c> is specified (this is the default), the process will never go into hibernation. </item> </taglist> diff --git a/lib/ssl/doc/src/ssl_app.xml b/lib/ssl/doc/src/ssl_app.xml index 84ad483617..0ee5b23e47 100644 --- a/lib/ssl/doc/src/ssl_app.xml +++ b/lib/ssl/doc/src/ssl_app.xml @@ -41,8 +41,9 @@ <section> <title>ENVIRONMENT</title> <p>The following application environment configuration parameters - are defined for the SSL application. Refer to application(3) for - more information about configuration parameters. + are defined for the SSL application. See <seealso + marker="kernel:application">application(3)</seealso>for more + information about configuration parameters. </p> <p>Note that the environment parameters can be set on the command line, for instance,</p> @@ -87,7 +88,7 @@ <section> <title>SEE ALSO</title> - <p>application(3)</p> + <p><seealso marker="kernel:application">application(3)</seealso></p> </section> </appref> diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index 76e14860ec..a8a494b2fc 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,12 +1,14 @@ %% -*- erlang -*- {"%VSN%", [ + {<<"5.2">>, [{restart_application, ssl}]}, {<<"5.1\\*">>, [{restart_application, ssl}]}, {<<"5.0\\*">>, [{restart_application, ssl}]}, {<<"4\\.*">>, [{restart_application, ssl}]}, {<<"3\\.*">>, [{restart_application, ssl}]} ], [ + {<<"5.2">>, [{restart_application, ssl}]}, {<<"5.1\\*">>, [{restart_application, ssl}]}, {<<"5.0\\*">>, [{restart_application, ssl}]}, {<<"4\\.*">>, [{restart_application, ssl}]}, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 647daeb1ac..fc06b5f1b0 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -162,7 +162,7 @@ connect(Host, Port, Options, Timeout) -> %% Description: Creates an ssl listen socket. %%-------------------------------------------------------------------- listen(_Port, []) -> - {error, enooptions}; + {error, nooptions}; listen(Port, Options0) -> try {ok, Config} = handle_options(Options0, server), @@ -380,13 +380,13 @@ getopts(#sslsocket{pid = {ListenSocket, #config{cb = {Transport,_,_,_}}}}, {ok, _} = Result -> Result; {error, InetError} -> - {error, {eoptions, {socket_options, OptionTags, InetError}}} + {error, {options, {socket_options, OptionTags, InetError}}} catch _:_ -> - {error, {eoptions, {socket_options, OptionTags}}} + {error, {options, {socket_options, OptionTags}}} end; getopts(#sslsocket{}, OptionTags) -> - {error, {eoptions, {socket_options, OptionTags}}}. + {error, {options, {socket_options, OptionTags}}}. %%-------------------------------------------------------------------- -spec setopts(#sslsocket{}, [gen_tcp:option()]) -> ok | {error, reason()}. @@ -400,7 +400,7 @@ setopts(#sslsocket{pid = Pid}, Options0) when is_pid(Pid), is_list(Options0) -> ssl_connection:set_opts(Pid, Options) catch _:_ -> - {error, {eoptions, {not_a_proplist, Options0}}} + {error, {options, {not_a_proplist, Options0}}} end; setopts(#sslsocket{pid = {ListenSocket, #config{cb = {Transport,_,_,_}}}}, Options) when is_list(Options) -> @@ -408,13 +408,13 @@ setopts(#sslsocket{pid = {ListenSocket, #config{cb = {Transport,_,_,_}}}}, Optio ok -> ok; {error, InetError} -> - {error, {eoptions, {socket_options, Options, InetError}}} + {error, {options, {socket_options, Options, InetError}}} catch _:Error -> - {error, {eoptions, {socket_options, Options, Error}}} + {error, {options, {socket_options, Options, Error}}} end; setopts(#sslsocket{}, Options) -> - {error, {eoptions,{not_a_proplist, Options}}}. + {error, {options,{not_a_proplist, Options}}}. %%--------------------------------------------------------------- -spec shutdown(#sslsocket{}, read | write | read_write) -> ok | {error, reason()}. @@ -503,24 +503,26 @@ format_error({error, Reason}) -> format_error(Reason) when is_list(Reason) -> Reason; format_error(closed) -> - "The connection is closed"; -format_error({ecacertfile, _}) -> - "Own CA certificate file is invalid."; -format_error({ecertfile, _}) -> - "Own certificate file is invalid."; -format_error({ekeyfile, _}) -> - "Own private key file is invalid."; -format_error({essl, Description}) -> - Description; -format_error({eoptions, Options}) -> - lists:flatten(io_lib:format("Error in options list: ~p~n", [Options])); + "TLS connection is closed"; +format_error({tls_alert, Description}) -> + "TLS Alert: " ++ Description; +format_error({options,{FileType, File, Reason}}) when FileType == cacertfile; + FileType == certfile; + FileType == keyfile; + FileType == dhfile -> + Error = file_error_format(Reason), + file_desc(FileType) ++ File ++ ": " ++ Error; +format_error({options, {socket_options, Option, Error}}) -> + lists:flatten(io_lib:format("Invalid transport socket option ~p: ~s", [Option, format_error(Error)])); +format_error({options, {socket_options, Option}}) -> + lists:flatten(io_lib:format("Invalid socket option: ~p", [Option])); +format_error({options, Options}) -> + lists:flatten(io_lib:format("Invalid TLS option: ~p", [Options])); format_error(Error) -> - case (catch inet:format_error(Error)) of - "unkknown POSIX" ++ _ -> - no_format(Error); - {'EXIT', _} -> - no_format(Error); + case inet:format_error(Error) of + "unknown POSIX" ++ _ -> + unexpected_format(Error); Other -> Other end. @@ -541,8 +543,6 @@ random_bytes(N) -> crypto:rand_bytes(N) end. - - %%%-------------------------------------------------------------- %%% Internal functions %%%-------------------------------------------------------------------- @@ -559,11 +559,11 @@ do_connect(Address, Port, {error, Reason} catch exit:{function_clause, _} -> - {error, {eoptions, {cb_info, CbInfo}}}; + {error, {options, {cb_info, CbInfo}}}; exit:badarg -> - {error, {eoptions, {socket_options, UserOpts}}}; + {error, {options, {socket_options, UserOpts}}}; exit:{badarg, _} -> - {error, {eoptions, {socket_options, UserOpts}}} + {error, {options, {socket_options, UserOpts}}} end. handle_options(Opts0, _Role) -> @@ -607,13 +607,20 @@ handle_options(Opts0, _Role) -> {verify_peer, UserFailIfNoPeerCert, ca_cert_default(verify_peer, UserVerifyFun, CaCerts), UserVerifyFun}; Value -> - throw({error, {eoptions, {verify, Value}}}) + throw({error, {options, {verify, Value}}}) end, CertFile = handle_option(certfile, Opts, <<>>), + Versions = case handle_option(versions, Opts, []) of + [] -> + ssl_record:supported_protocol_versions(); + Vsns -> + [ssl_record:protocol_version(Vsn) || Vsn <- Vsns] + end, + SSLOptions = #ssl_options{ - versions = handle_option(versions, Opts, []), + versions = Versions, verify = validate_option(verify, Verify), verify_fun = VerifyFun, fail_if_no_peer_cert = FailIfNoPeerCert, @@ -754,9 +761,9 @@ validate_option(ciphers, Value) when is_list(Value) -> try cipher_suites(Version, Value) catch exit:_ -> - throw({error, {eoptions, {ciphers, Value}}}); + throw({error, {options, {ciphers, Value}}}); error:_-> - throw({error, {eoptions, {ciphers, Value}}}) + throw({error, {options, {ciphers, Value}}}) end; validate_option(reuse_session, Value) when is_function(Value) -> Value; @@ -781,7 +788,7 @@ validate_option(client_preferred_next_protocols = Opt, {Precedence, PreferredPro when is_list(PreferredProtocols) -> case ssl_record:highest_protocol_version([]) of {3,0} -> - throw({error, {eoptions, {not_supported_in_sslv3, {Opt, Value}}}}); + throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); _ -> validate_binary_list(client_preferred_next_protocols, PreferredProtocols), validate_npn_ordering(Precedence), @@ -792,7 +799,7 @@ validate_option(client_preferred_next_protocols = Opt, {Precedence, PreferredPro byte_size(Default) > 0, byte_size(Default) < 256 -> case ssl_record:highest_protocol_version([]) of {3,0} -> - throw({error, {eoptions, {not_supported_in_sslv3, {Opt, Value}}}}); + throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); _ -> validate_binary_list(client_preferred_next_protocols, PreferredProtocols), validate_npn_ordering(Precedence), @@ -804,7 +811,7 @@ validate_option(client_preferred_next_protocols, undefined) -> validate_option(next_protocols_advertised = Opt, Value) when is_list(Value) -> case ssl_record:highest_protocol_version([]) of {3,0} -> - throw({error, {eoptions, {not_supported_in_sslv3, {Opt, Value}}}}); + throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); _ -> validate_binary_list(next_protocols_advertised, Value), Value @@ -813,14 +820,14 @@ validate_option(next_protocols_advertised = Opt, Value) when is_list(Value) -> validate_option(next_protocols_advertised, undefined) -> undefined; validate_option(Opt, Value) -> - throw({error, {eoptions, {Opt, Value}}}). + throw({error, {options, {Opt, Value}}}). validate_npn_ordering(client) -> ok; validate_npn_ordering(server) -> ok; validate_npn_ordering(Value) -> - throw({error, {eoptions, {client_preferred_next_protocols, {invalid_precedence, Value}}}}). + throw({error, {options, {client_preferred_next_protocols, {invalid_precedence, Value}}}}). validate_binary_list(Opt, List) -> lists:foreach( @@ -829,7 +836,7 @@ validate_binary_list(Opt, List) -> byte_size(Bin) < 256 -> ok; (Bin) -> - throw({error, {eoptions, {Opt, {invalid_protocol, Bin}}}}) + throw({error, {options, {Opt, {invalid_protocol, Bin}}}}) end, List). validate_versions([], Versions) -> @@ -840,23 +847,23 @@ validate_versions([Version | Rest], Versions) when Version == 'tlsv1.2'; Version == sslv3 -> validate_versions(Rest, Versions); validate_versions([Ver| _], Versions) -> - throw({error, {eoptions, {Ver, {versions, Versions}}}}). + throw({error, {options, {Ver, {versions, Versions}}}}). validate_inet_option(mode, Value) when Value =/= list, Value =/= binary -> - throw({error, {eoptions, {mode,Value}}}); + throw({error, {options, {mode,Value}}}); validate_inet_option(packet, Value) when not (is_atom(Value) orelse is_integer(Value)) -> - throw({error, {eoptions, {packet,Value}}}); + throw({error, {options, {packet,Value}}}); validate_inet_option(packet_size, Value) when not is_integer(Value) -> - throw({error, {eoptions, {packet_size,Value}}}); + throw({error, {options, {packet_size,Value}}}); validate_inet_option(header, Value) when not is_integer(Value) -> - throw({error, {eoptions, {header,Value}}}); + throw({error, {options, {header,Value}}}); validate_inet_option(active, Value) when Value =/= true, Value =/= false, Value =/= once -> - throw({error, {eoptions, {active,Value}}}); + throw({error, {options, {active,Value}}}); validate_inet_option(_, _) -> ok. @@ -935,8 +942,27 @@ cipher_suites(Version, Ciphers0) -> Ciphers = [ssl_cipher:openssl_suite(C) || C <- string:tokens(Ciphers0, ":")], cipher_suites(Version, Ciphers). -no_format(Error) -> - lists:flatten(io_lib:format("No format string for error: \"~p\" available.", [Error])). +unexpected_format(Error) -> + lists:flatten(io_lib:format("Unexpected error: ~p", [Error])). + +file_error_format({error, Error})-> + case file:format_error(Error) of + "unknown POSIX error" -> + "decoding error"; + Str -> + Str + end; +file_error_format(_) -> + "decoding error". + +file_desc(cacertfile) -> + "Invalid CA certificate file "; +file_desc(certfile) -> + "Invalid certificate file "; +file_desc(keyfile) -> + "Invalid key file "; +file_desc(dhfile) -> + "Invalid DH params file ". detect(_Pred, []) -> undefined; diff --git a/lib/ssl/src/ssl_alert.erl b/lib/ssl/src/ssl_alert.erl index f94a1136a0..94e95d3cd3 100644 --- a/lib/ssl/src/ssl_alert.erl +++ b/lib/ssl/src/ssl_alert.erl @@ -45,7 +45,7 @@ reason_code(#alert{description = ?CLOSE_NOTIFY}, _) -> closed; reason_code(#alert{description = Description}, _) -> - {essl, description_txt(Description)}. + {tls_alert, description_txt(Description)}. %%-------------------------------------------------------------------- -spec alert_txt(#alert{}) -> string(). diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index f51f1c6115..4d29ecce7a 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -73,7 +73,6 @@ session_cache, % session_cache_cb, % negotiated_version, % tls_version() - supported_protocol_versions, % [atom()] client_certificate_requested = false, key_algorithm, % atom as defined by cipher_suite hashsign_algorithm, % atom as defined by cipher_suite @@ -472,6 +471,13 @@ abbreviated(#finished{verify_data = Data} = Finished, handle_own_alert(Alert, Version, abbreviated, State) end; +%% only allowed to send next_protocol message after change cipher spec +%% & before finished message and it is not allowed during renegotiation +abbreviated(#next_protocol{selected_protocol = SelectedProtocol}, + #state{role = server, expecting_next_protocol_negotiation = true} = State0) -> + {Record, State} = next_record(State0#state{next_protocol = SelectedProtocol}), + next_state(abbreviated, abbreviated, Record, State); + abbreviated(timeout, State) -> { next_state, abbreviated, State, hibernate }; @@ -656,11 +662,10 @@ cipher(#certificate_verify{signature = Signature, hashsign_algorithm = CertHashS handle_own_alert(Alert, Version, cipher, State0) end; -% client must send a next protocol message if we are expecting it +%% client must send a next protocol message if we are expecting it cipher(#finished{}, #state{role = server, expecting_next_protocol_negotiation = true, next_protocol = undefined, negotiated_version = Version} = State0) -> - handle_own_alert(?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE), Version, cipher, State0), - {stop, normal, State0}; + handle_own_alert(?ALERT_REC(?FATAL,?UNEXPECTED_MESSAGE), Version, cipher, State0); cipher(#finished{verify_data = Data} = Finished, #state{negotiated_version = Version, @@ -682,8 +687,8 @@ cipher(#finished{verify_data = Data} = Finished, handle_own_alert(Alert, Version, cipher, State) end; -% only allowed to send next_protocol message after change cipher spec -% & before finished message and it is not allowed during renegotiation +%% only allowed to send next_protocol message after change cipher spec +%% & before finished message and it is not allowed during renegotiation cipher(#next_protocol{selected_protocol = SelectedProtocol}, #state{role = server, expecting_next_protocol_negotiation = true} = State0) -> {Record, State} = next_record(State0#state{next_protocol = SelectedProtocol}), @@ -987,7 +992,7 @@ handle_info({ErrorTag, Socket, econnaborted}, StateName, #state{socket = Socket, transport_cb = Transport, start_or_recv_from = StartFrom, role = Role, error_tag = ErrorTag} = State) when StateName =/= connection -> - alert_user(Transport, Socket, StartFrom, ?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE), Role), + alert_user(Transport, Socket, StartFrom, ?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), Role), {stop, normal, State}; handle_info({ErrorTag, Socket, Reason}, StateName, #state{socket = Socket, @@ -1136,9 +1141,8 @@ init_certificates(#ssl_options{cacerts = CaCerts, end, {ok, _, _, _, _, _} = ssl_manager:connection_init(Certs, Role) catch - Error:Reason -> - handle_file_error(?LINE, Error, Reason, CACertFile, {ecacertfile, Reason}, - erlang:get_stacktrace()) + _:Reason -> + file_error(CACertFile, {cacertfile, Reason}) end, init_certificates(Cert, CertDbRef, CertDbHandle, FileRefHandle, PemCacheHandle, CacheHandle, CertFile, Role). @@ -1158,9 +1162,8 @@ init_certificates(undefined, CertDbRef, CertDbHandle, FileRefHandle, PemCacheHan [OwnCert] = ssl_certificate:file_to_certificats(CertFile, PemCacheHandle), {ok, CertDbRef, CertDbHandle, FileRefHandle, PemCacheHandle, CacheRef, OwnCert} catch - Error:Reason -> - handle_file_error(?LINE, Error, Reason, CertFile, {ecertfile, Reason}, - erlang:get_stacktrace()) + _:Reason -> + file_error(CertFile, {certfile, Reason}) end; init_certificates(Cert, CertDbRef, CertDbHandle, FileRefHandle, PemCacheHandle, CacheRef, _, _) -> {ok, CertDbRef, CertDbHandle, FileRefHandle, PemCacheHandle, CacheRef, Cert}. @@ -1177,9 +1180,8 @@ init_private_key(DbHandle, undefined, KeyFile, Password, _) -> ], private_key(public_key:pem_entry_decode(PemEntry, Password)) catch - Error:Reason -> - handle_file_error(?LINE, Error, Reason, KeyFile, {ekeyfile, Reason}, - erlang:get_stacktrace()) + _:Reason -> + file_error(KeyFile, {keyfile, Reason}) end; %% First two clauses are for backwards compatibility @@ -1205,18 +1207,14 @@ private_key(#'PrivateKeyInfo'{privateKeyAlgorithm = private_key(Key) -> Key. --spec(handle_file_error(_,_,_,_,_,_) -> no_return()). -handle_file_error(Line, Error, {badmatch, Reason}, File, Throw, Stack) -> - file_error(Line, Error, Reason, File, Throw, Stack); -handle_file_error(Line, Error, Reason, File, Throw, Stack) -> - file_error(Line, Error, Reason, File, Throw, Stack). - --spec(file_error(_,_,_,_,_,_) -> no_return()). -file_error(Line, Error, Reason, File, Throw, Stack) -> - Report = io_lib:format("SSL: ~p: ~p:~p ~s~n ~p~n", - [Line, Error, Reason, File, Stack]), - error_logger:error_report(Report), - throw(Throw). +-spec(file_error(_,_) -> no_return()). +file_error(File, Throw) -> + case Throw of + {Opt,{badmatch, {error, {badmatch, Error}}}} -> + throw({options, {Opt, binary_to_list(File), Error}}); + _ -> + throw(Throw) + end. init_diffie_hellman(_,Params, _,_) when is_binary(Params)-> public_key:der_decode('DHParameter', Params); @@ -1234,9 +1232,8 @@ init_diffie_hellman(DbHandle,_, DHParamFile, server) -> ?DEFAULT_DIFFIE_HELLMAN_PARAMS end catch - Error:Reason -> - handle_file_error(?LINE, Error, Reason, - DHParamFile, {edhfile, Reason}, erlang:get_stacktrace()) + _:Reason -> + file_error(DHParamFile, {dhfile, Reason}) end. sync_send_all_state_event(FsmPid, Event) -> @@ -2179,13 +2176,13 @@ get_socket_opts(Transport, Socket, [Tag | Tags], SockOpts, Acc) -> {ok, [Opt]} -> get_socket_opts(Transport, Socket, Tags, SockOpts, [Opt | Acc]); {error, Error} -> - {error, {eoptions, {socket_option, Tag, Error}}} + {error, {options, {socket_options, Tag, Error}}} catch %% So that inet behavior does not crash our process - _:Error -> {error, {eoptions, {socket_option, Tag, Error}}} + _:Error -> {error, {options, {socket_options, Tag, Error}}} end; get_socket_opts(_, _,Opts, _,_) -> - {error, {eoptions, {socket_options, Opts, function_clause}}}. + {error, {options, {socket_options, Opts, function_clause}}}. set_socket_opts(_,_, [], SockOpts, []) -> {ok, SockOpts}; @@ -2195,18 +2192,18 @@ set_socket_opts(Transport, Socket, [], SockOpts, Other) -> ok -> {ok, SockOpts}; {error, InetError} -> - {{error, {eoptions, {socket_option, Other, InetError}}}, SockOpts} + {{error, {options, {socket_options, Other, InetError}}}, SockOpts} catch _:Error -> %% So that inet behavior does not crash our process - {{error, {eoptions, {socket_option, Other, Error}}}, SockOpts} + {{error, {options, {socket_options, Other, Error}}}, SockOpts} end; set_socket_opts(Transport,Socket, [{mode, Mode}| Opts], SockOpts, Other) when Mode == list; Mode == binary -> set_socket_opts(Transport, Socket, Opts, SockOpts#socket_options{mode = Mode}, Other); set_socket_opts(_, _, [{mode, _} = Opt| _], SockOpts, _) -> - {{error, {eoptions, {socket_option, Opt}}}, SockOpts}; + {{error, {options, {socket_options, Opt}}}, SockOpts}; set_socket_opts(Transport,Socket, [{packet, Packet}| Opts], SockOpts, Other) when Packet == raw; Packet == 0; Packet == 1; @@ -2225,19 +2222,19 @@ set_socket_opts(Transport,Socket, [{packet, Packet}| Opts], SockOpts, Other) whe set_socket_opts(Transport, Socket, Opts, SockOpts#socket_options{packet = Packet}, Other); set_socket_opts(_, _, [{packet, _} = Opt| _], SockOpts, _) -> - {{error, {eoptions, {socket_option, Opt}}}, SockOpts}; + {{error, {options, {socket_options, Opt}}}, SockOpts}; set_socket_opts(Transport, Socket, [{header, Header}| Opts], SockOpts, Other) when is_integer(Header) -> set_socket_opts(Transport, Socket, Opts, SockOpts#socket_options{header = Header}, Other); set_socket_opts(_, _, [{header, _} = Opt| _], SockOpts, _) -> - {{error,{eoptions, {socket_option, Opt}}}, SockOpts}; + {{error,{options, {socket_options, Opt}}}, SockOpts}; set_socket_opts(Transport, Socket, [{active, Active}| Opts], SockOpts, Other) when Active == once; Active == true; Active == false -> set_socket_opts(Transport, Socket, Opts, SockOpts#socket_options{active = Active}, Other); set_socket_opts(_, _, [{active, _} = Opt| _], SockOpts, _) -> - {{error, {eoptions, {socket_option, Opt}} }, SockOpts}; + {{error, {options, {socket_options, Opt}} }, SockOpts}; set_socket_opts(Transport, Socket, [Opt | Opts], SockOpts, Other) -> set_socket_opts(Transport, Socket, Opts, SockOpts, [Opt | Other]). diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 1929370991..889d310ca8 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2012. All Rights Reserved. +%% Copyright Ericsson AB 2007-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -61,11 +61,7 @@ client_hello(Host, Port, ConnectionStates, ciphers = UserSuites } = SslOpts, Cache, CacheCb, Renegotiation, OwnCert) -> - - Fun = fun(Version) -> - ssl_record:protocol_version(Version) - end, - Version = ssl_record:highest_protocol_version(lists:map(Fun, Versions)), + Version = ssl_record:highest_protocol_version(Versions), Pending = ssl_record:pending_connection_state(ConnectionStates, read), SecParams = Pending#connection_state.security_parameters, Ciphers = available_suites(UserSuites, Version), @@ -139,10 +135,11 @@ hello(#server_hello{cipher_suite = CipherSuite, server_version = Version, compression_method = Compression, random = Random, session_id = SessionId, renegotiation_info = Info, hash_signs = _HashSigns} = Hello, - #ssl_options{secure_renegotiate = SecureRenegotation, next_protocol_selector = NextProtocolSelector}, + #ssl_options{secure_renegotiate = SecureRenegotation, next_protocol_selector = NextProtocolSelector, + versions = SupportedVersions}, ConnectionStates0, Renegotiation) -> %%TODO: select hash and signature algorigthm - case ssl_record:is_acceptable_version(Version) of + case ssl_record:is_acceptable_version(Version, SupportedVersions) of true -> case handle_renegotiation_info(client, Info, ConnectionStates0, Renegotiation, SecureRenegotation, []) of @@ -171,7 +168,7 @@ hello(#client_hello{client_version = ClientVersion, random = Random, {Port, Session0, Cache, CacheCb, ConnectionStates0, Cert}, Renegotiation) -> %% TODO: select hash and signature algorithm Version = select_version(ClientVersion, Versions), - case ssl_record:is_acceptable_version(Version) of + case ssl_record:is_acceptable_version(Version, Versions) of true -> {Type, #session{cipher_suite = CipherSuite, compression_method = Compression} = Session} @@ -869,11 +866,7 @@ hello_security_parameters(server, Version, ConnectionState, CipherSuite, Random, }. select_version(ClientVersion, Versions) -> - Fun = fun(Version) -> - ssl_record:protocol_version(Version) - end, - ServerVersion = ssl_record:highest_protocol_version(lists:map(Fun, - Versions)), + ServerVersion = ssl_record:highest_protocol_version(Versions), ssl_record:lowest_protocol_version(ClientVersion, ServerVersion). select_cipher_suite([], _) -> diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 173b9611c6..26aca56739 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2012. All Rights Reserved. +%% Copyright Ericsson AB 2007-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -56,7 +56,7 @@ %% Misc. -export([protocol_version/1, lowest_protocol_version/2, highest_protocol_version/1, supported_protocol_versions/0, - is_acceptable_version/1]). + is_acceptable_version/1, is_acceptable_version/2]). -export([compressions/0]). @@ -475,8 +475,10 @@ supported_protocol_versions([_|_] = Vsns) -> %%-------------------------------------------------------------------- -spec is_acceptable_version(tls_version()) -> boolean(). +-spec is_acceptable_version(tls_version(), Supported :: [tls_version()]) -> boolean(). %% %% Description: ssl version 2 is not acceptable security risks are too big. +%% %%-------------------------------------------------------------------- is_acceptable_version({N,_}) when N >= ?LOWEST_MAJOR_SUPPORTED_VERSION -> @@ -484,6 +486,12 @@ is_acceptable_version({N,_}) is_acceptable_version(_) -> false. +is_acceptable_version({N,_} = Version, Versions) + when N >= ?LOWEST_MAJOR_SUPPORTED_VERSION -> + lists:member(Version, Versions); +is_acceptable_version(_,_) -> + false. + %%-------------------------------------------------------------------- -spec compressions() -> [binary()]. %% diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index db203a47c4..6b8f226a77 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -99,10 +99,10 @@ options_tests() -> invalid_inet_set_option_not_list, invalid_inet_set_option_improper_list, dh_params, - ecertfile, - ecacertfile, - ekeyfile, - eoptions, + invalid_certfile, + invalid_cacertfile, + invalid_keyfile, + invalid_options, protocol_versions, empty_protocol_versions, ipv6, @@ -126,7 +126,8 @@ api_tests() -> hibernate, listen_socket, ssl_accept_timeout, - ssl_recv_timeout + ssl_recv_timeout, + versions_option ]. session_tests() -> @@ -822,7 +823,7 @@ invalid_inet_get_option_not_list(Config) when is_list(Config) -> get_invalid_inet_option_not_list(Socket) -> - {error, {eoptions, {socket_options, some_invalid_atom_here}}} + {error, {options, {socket_options, some_invalid_atom_here}}} = ssl:getopts(Socket, some_invalid_atom_here), ok. @@ -854,7 +855,7 @@ invalid_inet_get_option_improper_list(Config) when is_list(Config) -> get_invalid_inet_option_improper_list(Socket) -> - {error, {eoptions, {socket_options, foo,_}}} = ssl:getopts(Socket, [packet | foo]), + {error, {options, {socket_options, foo,_}}} = ssl:getopts(Socket, [packet | foo]), ok. %%-------------------------------------------------------------------- @@ -884,10 +885,10 @@ invalid_inet_set_option(Config) when is_list(Config) -> ssl_test_lib:close(Client). set_invalid_inet_option(Socket) -> - {error, {eoptions, {socket_option, {packet, foo}}}} = ssl:setopts(Socket, [{packet, foo}]), - {error, {eoptions, {socket_option, {header, foo}}}} = ssl:setopts(Socket, [{header, foo}]), - {error, {eoptions, {socket_option, {active, foo}}}} = ssl:setopts(Socket, [{active, foo}]), - {error, {eoptions, {socket_option, {mode, foo}}}} = ssl:setopts(Socket, [{mode, foo}]), + {error, {options, {socket_options, {packet, foo}}}} = ssl:setopts(Socket, [{packet, foo}]), + {error, {options, {socket_options, {header, foo}}}} = ssl:setopts(Socket, [{header, foo}]), + {error, {options, {socket_options, {active, foo}}}} = ssl:setopts(Socket, [{active, foo}]), + {error, {options, {socket_options, {mode, foo}}}} = ssl:setopts(Socket, [{mode, foo}]), ok. %%-------------------------------------------------------------------- invalid_inet_set_option_not_list() -> @@ -917,7 +918,7 @@ invalid_inet_set_option_not_list(Config) when is_list(Config) -> set_invalid_inet_option_not_list(Socket) -> - {error, {eoptions, {not_a_proplist, some_invalid_atom_here}}} + {error, {options, {not_a_proplist, some_invalid_atom_here}}} = ssl:setopts(Socket, some_invalid_atom_here), ok. @@ -948,7 +949,7 @@ invalid_inet_set_option_improper_list(Config) when is_list(Config) -> ssl_test_lib:close(Client). set_invalid_inet_option_improper_list(Socket) -> - {error, {eoptions, {not_a_proplist, [{packet, 0} | {foo, 2}]}}} = + {error, {options, {not_a_proplist, [{packet, 0} | {foo, 2}]}}} = ssl:setopts(Socket, [{packet, 0} | {foo, 2}]), ok. @@ -1194,12 +1195,12 @@ tcp_connect(Config) when is_list(Config) -> {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), TcpOpts = [binary, {reuseaddr, true}], - Server = ssl_test_lib:start_upgrade_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {timeout, 5000}, - {mfa, {?MODULE, dummy, []}}, - {tcp_options, TcpOpts}, - {ssl_options, ServerOpts}]), + Server = ssl_test_lib:start_upgrade_server_error([{node, ServerNode}, {port, 0}, + {from, self()}, + {timeout, 5000}, + {mfa, {?MODULE, dummy, []}}, + {tcp_options, TcpOpts}, + {ssl_options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), {ok, Socket} = gen_tcp:connect(Hostname, Port, [binary, {packet, 0}]), @@ -1222,12 +1223,12 @@ tcp_connect_big(Config) when is_list(Config) -> {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), TcpOpts = [binary, {reuseaddr, true}], - Server = ssl_test_lib:start_upgrade_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {timeout, 5000}, - {mfa, {?MODULE, dummy, []}}, - {tcp_options, TcpOpts}, - {ssl_options, ServerOpts}]), + Server = ssl_test_lib:start_upgrade_server_error([{node, ServerNode}, {port, 0}, + {from, self()}, + {timeout, 5000}, + {mfa, {?MODULE, dummy, []}}, + {tcp_options, TcpOpts}, + {ssl_options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), {ok, Socket} = gen_tcp:connect(Hostname, Port, [binary, {packet, 0}]), @@ -1286,9 +1287,9 @@ ipv6(Config) when is_list(Config) -> %%-------------------------------------------------------------------- -ekeyfile() -> +invalid_keyfile() -> [{doc,"Test what happens with an invalid key file"}]. -ekeyfile(Config) when is_list(Config) -> +invalid_keyfile(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), BadOpts = ?config(server_bad_key, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -1304,16 +1305,17 @@ ekeyfile(Config) when is_list(Config) -> ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {options, ClientOpts}]), - - ssl_test_lib:check_result(Server, {error, ekeyfile}, Client, - {error, closed}). + + File = proplists:get_value(keyfile,BadOpts), + ssl_test_lib:check_result(Server, {error,{options, {keyfile, File, {error,enoent}}}}, Client, + {error, closed}). %%-------------------------------------------------------------------- -ecertfile() -> +invalid_certfile() -> [{doc,"Test what happens with an invalid cert file"}]. -ecertfile(Config) when is_list(Config) -> +invalid_certfile(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), ServerBadOpts = ?config(server_bad_cert, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -1330,16 +1332,16 @@ ecertfile(Config) when is_list(Config) -> {port, Port}, {host, Hostname}, {from, self()}, {options, ClientOpts}]), - - ssl_test_lib:check_result(Server, {error, ecertfile}, Client, - {error, closed}). + File = proplists:get_value(certfile, ServerBadOpts), + ssl_test_lib:check_result(Server, {error,{options, {certfile, File, {error,enoent}}}}, + Client, {error, closed}). %%-------------------------------------------------------------------- -ecacertfile() -> +invalid_cacertfile() -> [{doc,"Test what happens with an invalid cacert file"}]. -ecacertfile(Config) when is_list(Config) -> +invalid_cacertfile(Config) when is_list(Config) -> ClientOpts = [{reuseaddr, true}|?config(client_opts, Config)], ServerBadOpts = [{reuseaddr, true}|?config(server_bad_ca, Config)], {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -1357,11 +1359,12 @@ ecacertfile(Config) when is_list(Config) -> {port, Port0}, {host, Hostname}, {from, self()}, {options, ClientOpts}]), + + File0 = proplists:get_value(cacertfile, ServerBadOpts), - ssl_test_lib:check_result(Server0, {error, ecacertfile}, + ssl_test_lib:check_result(Server0, {error, {options, {cacertfile, File0,{error,enoent}}}}, Client0, {error, closed}), - File0 = proplists:get_value(cacertfile, ServerBadOpts), File = File0 ++ "do_not_exit.pem", ServerBadOpts1 = [{cacertfile, File}|proplists:delete(cacertfile, ServerBadOpts)], @@ -1378,31 +1381,32 @@ ecacertfile(Config) when is_list(Config) -> {from, self()}, {options, ClientOpts}]), - ssl_test_lib:check_result(Server1, {error, ecacertfile}, + + ssl_test_lib:check_result(Server1, {error, {options, {cacertfile, File,{error,enoent}}}}, Client1, {error, closed}), ok. %%-------------------------------------------------------------------- -eoptions() -> +invalid_options() -> [{doc,"Test what happens when we give invalid options"}]. -eoptions(Config) when is_list(Config) -> +invalid_options(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), ServerOpts = ?config(server_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Check = fun(Client, Server, {versions, [sslv2, sslv3]} = Option) -> ssl_test_lib:check_result(Server, - {error, {eoptions, {sslv2, Option}}}, + {error, {options, {sslv2, Option}}}, Client, - {error, {eoptions, {sslv2, Option}}}); + {error, {options, {sslv2, Option}}}); (Client, Server, Option) -> ssl_test_lib:check_result(Server, - {error, {eoptions, Option}}, + {error, {options, Option}}, Client, - {error, {eoptions, Option}}) + {error, {options, Option}}) end, TestOpts = [{versions, [sslv2, sslv3]}, @@ -1593,8 +1597,8 @@ default_reject_anonymous(Config) when is_list(Config) -> [{ciphers,[Cipher]} | ClientOpts]}]), - ssl_test_lib:check_result(Server, {error, {essl, "insufficient security"}}, - Client, {error, {essl, "insufficient security"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "insufficient security"}}, + Client, {error, {tls_alert, "insufficient security"}}). %%-------------------------------------------------------------------- reuse_session() -> @@ -2656,6 +2660,42 @@ session_cache_process_mnesia(Config) when is_list(Config) -> session_cache_process(mnesia,Config). %%-------------------------------------------------------------------- + +versions_option() -> + [{doc,"Test API versions option to connect/listen."}]. +versions_option(Config) when is_list(Config) -> + ClientOpts = ?config(client_opts, Config), + ServerOpts = ?config(server_opts, Config), + + Supported = proplists:get_value(supported, ssl:versions()), + Available = proplists:get_value(available, ssl:versions()), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{versions, Supported} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + Server ! listen, + + ErrClient = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {options, [{versions , Available -- Supported} | ClientOpts]}]), + receive + {Server, _} -> + ok + end, + + ssl_test_lib:check_result(ErrClient, {error, {tls_alert, "protocol version"}}). +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- send_recv_result(Socket) -> @@ -3147,7 +3187,7 @@ treashold(N, _) -> N + 1. get_invalid_inet_option(Socket) -> - {error, {eoptions, {socket_option, foo, _}}} = ssl:getopts(Socket, [foo]), + {error, {options, {socket_options, foo, _}}} = ssl:getopts(Socket, [foo]), ok. shutdown_result(Socket, server) -> diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 86e1d47be7..26938bda50 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -252,8 +252,8 @@ server_require_peer_cert_fail(Config) when is_list(Config) -> {from, self()}, {options, [{active, false} | BadClientOpts]}]), - ssl_test_lib:check_result(Server, {error, {essl, "handshake failure"}}, - Client, {error, {essl, "handshake failure"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "handshake failure"}}, + Client, {error, {tls_alert, "handshake failure"}}). %%-------------------------------------------------------------------- @@ -293,14 +293,14 @@ verify_fun_always_run_client(Config) when is_list(Config) -> [{verify, verify_peer}, {verify_fun, FunAndState} | ClientOpts]}]), - %% Server error may be {essl,"handshake failure"} or closed depending on timing + %% Server error may be {tls_alert,"handshake failure"} or closed depending on timing %% this is not a bug it is a circumstance of how tcp works! receive {Server, ServerError} -> ct:print("Server Error ~p~n", [ServerError]) end, - ssl_test_lib:check_result(Client, {error, {essl, "handshake failure"}}). + ssl_test_lib:check_result(Client, {error, {tls_alert, "handshake failure"}}). %%-------------------------------------------------------------------- verify_fun_always_run_server() -> @@ -342,14 +342,14 @@ verify_fun_always_run_server(Config) when is_list(Config) -> [{verify, verify_peer} | ClientOpts]}]), - %% Client error may be {essl, "handshake failure" } or closed depending on timing + %% Client error may be {tls_alert, "handshake failure" } or closed depending on timing %% this is not a bug it is a circumstance of how tcp works! receive {Client, ClientError} -> ct:print("Client Error ~p~n", [ClientError]) end, - ssl_test_lib:check_result(Server, {error, {essl, "handshake failure"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "handshake failure"}}). %%-------------------------------------------------------------------- @@ -432,8 +432,8 @@ cert_expired(Config) when is_list(Config) -> {from, self()}, {options, [{verify, verify_peer} | ClientOpts]}]), - ssl_test_lib:check_result(Server, {error, {essl, "certificate expired"}}, - Client, {error, {essl, "certificate expired"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "certificate expired"}}, + Client, {error, {tls_alert, "certificate expired"}}). two_digits_str(N) when N < 10 -> lists:flatten(io_lib:format("0~p", [N])); @@ -710,8 +710,8 @@ invalid_signature_server(Config) when is_list(Config) -> {from, self()}, {options, [{verify, verify_peer} | ClientOpts]}]), - tcp_delivery_workaround(Server, {error, {essl, "bad certificate"}}, - Client, {error, {essl, "bad certificate"}}). + tcp_delivery_workaround(Server, {error, {tls_alert, "bad certificate"}}, + Client, {error, {tls_alert, "bad certificate"}}). %%-------------------------------------------------------------------- @@ -747,8 +747,8 @@ invalid_signature_client(Config) when is_list(Config) -> {from, self()}, {options, NewClientOpts}]), - tcp_delivery_workaround(Server, {error, {essl, "bad certificate"}}, - Client, {error, {essl, "bad certificate"}}). + tcp_delivery_workaround(Server, {error, {tls_alert, "bad certificate"}}, + Client, {error, {tls_alert, "bad certificate"}}). %%-------------------------------------------------------------------- @@ -792,7 +792,7 @@ server_verify_no_cacerts(Config) when is_list(Config) -> {options, [{verify, verify_peer} | ServerOpts]}]), - ssl_test_lib:check_result(Server, {error, {eoptions, {cacertfile, ""}}}). + ssl_test_lib:check_result(Server, {error, {options, {cacertfile, ""}}}). %%-------------------------------------------------------------------- @@ -829,8 +829,8 @@ unknown_server_ca_fail(Config) when is_list(Config) -> {verify_fun, FunAndState} | ClientOpts]}]), - ssl_test_lib:check_result(Server, {error, {essl, "unknown ca"}}, - Client, {error, {essl, "unknown ca"}}). + ssl_test_lib:check_result(Server, {error, {tls_alert, "unknown ca"}}, + Client, {error, {tls_alert, "unknown ca"}}). %%-------------------------------------------------------------------- unknown_server_ca_accept_verify_none() -> diff --git a/lib/ssl/test/ssl_npn_handshake_SUITE.erl b/lib/ssl/test/ssl_npn_handshake_SUITE.erl index 4e848095a5..8c1b22cf5e 100644 --- a/lib/ssl/test/ssl_npn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_npn_handshake_SUITE.erl @@ -24,6 +24,7 @@ -compile(export_all). -include_lib("common_test/include/ct.hrl"). +-define(SLEEP, 500). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- @@ -55,7 +56,8 @@ next_protocol_tests() -> fallback_npn_handshake_server_preference, client_negotiate_server_does_not_support, no_client_negotiate_but_server_supports_npn, - renegotiate_from_client_after_npn_handshake + renegotiate_from_client_after_npn_handshake, + npn_handshake_session_reused ]. next_protocol_not_supported() -> @@ -106,15 +108,15 @@ end_per_group(_GroupName, Config) -> %%-------------------------------------------------------------------- validate_empty_protocols_are_not_allowed(Config) when is_list(Config) -> - {error, {eoptions, {next_protocols_advertised, {invalid_protocol, <<>>}}}} + {error, {options, {next_protocols_advertised, {invalid_protocol, <<>>}}}} = (catch ssl:listen(9443, [{next_protocols_advertised, [<<"foo/1">>, <<"">>]}])), - {error, {eoptions, {client_preferred_next_protocols, {invalid_protocol, <<>>}}}} + {error, {options, {client_preferred_next_protocols, {invalid_protocol, <<>>}}}} = (catch ssl:connect({127,0,0,1}, 9443, [{client_preferred_next_protocols, {client, [<<"foo/1">>, <<"">>], <<"foox/1">>}}], infinity)), Option = {client_preferred_next_protocols, {invalid_protocol, <<"">>}}, - {error, {eoptions, Option}} = (catch ssl:connect({127,0,0,1}, 9443, [Option], infinity)). + {error, {options, Option}} = (catch ssl:connect({127,0,0,1}, 9443, [Option], infinity)). %-------------------------------------------------------------------------------- @@ -126,12 +128,12 @@ validate_empty_advertisement_list_is_allowed(Config) when is_list(Config) -> validate_advertisement_must_be_a_binary_list(Config) when is_list(Config) -> Option = {next_protocols_advertised, blah}, - {error, {eoptions, Option}} = (catch ssl:listen(9443, [Option])). + {error, {options, Option}} = (catch ssl:listen(9443, [Option])). %-------------------------------------------------------------------------------- validate_client_protocols_must_be_a_tuple(Config) when is_list(Config) -> Option = {client_preferred_next_protocols, [<<"foo/1">>]}, - {error, {eoptions, Option}} = (catch ssl:connect({127,0,0,1}, 9443, [Option])). + {error, {options, Option}} = (catch ssl:connect({127,0,0,1}, 9443, [Option])). %-------------------------------------------------------------------------------- @@ -220,7 +222,7 @@ npn_not_supported_client(Config) when is_list(Config) -> {from, self()}, {options, ClientOpts}]), ssl_test_lib:check_result(Client, {error, - {eoptions, + {options, {not_supported_in_sslv3, PrefProtocols}}}). %-------------------------------------------------------------------------------- @@ -229,7 +231,57 @@ npn_not_supported_server(Config) when is_list(Config)-> AdvProtocols = {next_protocols_advertised, [<<"spdy/2">>, <<"http/1.1">>, <<"http/1.0">>]}, ServerOpts = [AdvProtocols] ++ ServerOpts0, - {error, {eoptions, {not_supported_in_sslv3, AdvProtocols}}} = ssl:listen(0, ServerOpts). + {error, {options, {not_supported_in_sslv3, AdvProtocols}}} = ssl:listen(0, ServerOpts). + +%-------------------------------------------------------------------------------- +npn_handshake_session_reused(Config) when is_list(Config)-> + ClientOpts0 = ?config(client_opts, Config), + ClientOpts = [{client_preferred_next_protocols, + {client, [<<"http/1.0">>], <<"http/1.1">>}}] ++ ClientOpts0, + ServerOpts0 = ?config(server_opts, Config), + ServerOpts =[{next_protocols_advertised, + [<<"spdy/2">>, <<"http/1.1">>, <<"http/1.0">>]}] ++ ServerOpts0, + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, session_info_result, []}}, + {options, ServerOpts}]), + + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, no_result_msg, []}}, + {options, ClientOpts}]), + + SessionInfo = + receive + {Server, Info} -> + Info + end, + + Server ! {listen, {mfa, {ssl_test_lib, no_result, []}}}, + + %% Make sure session is registered + ct:sleep(?SLEEP), + + Client1 = + ssl_test_lib:start_client([{node, ClientNode}, + {port, Port}, {host, Hostname}, + {mfa, {ssl_test_lib, session_info_result, []}}, + {from, self()}, {options, ClientOpts}]), + + receive + {Client1, SessionInfo} -> + ok; + {Client1, Other} -> + ct:fail(Other) + end, + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client), + ssl_test_lib:close(Client1). %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/ssl/test/ssl_packet_SUITE.erl b/lib/ssl/test/ssl_packet_SUITE.erl index 158c40e372..4116bb39d1 100644 --- a/lib/ssl/test/ssl_packet_SUITE.erl +++ b/lib/ssl/test/ssl_packet_SUITE.erl @@ -55,6 +55,13 @@ all() -> {group, 'sslv3'} ]. +groups() -> + [{'tlsv1.2', [], packet_tests()}, + {'tlsv1.1', [], packet_tests()}, + {'tlsv1', [], packet_tests()}, + {'sslv3', [], packet_tests()} + ]. + packet_tests() -> active_packet_tests() ++ active_once_packet_tests() ++ passive_packet_tests() ++ [packet_send_to_large, @@ -133,7 +140,6 @@ init_per_suite(Config) -> try crypto:start() of ok -> application:start(public_key), - ssl:start(), Result = (catch make_certs:all(?config(data_dir, Config), ?config(priv_dir, Config))), @@ -184,7 +190,7 @@ packet_raw_passive_many_small() -> packet_raw_passive_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, raw}", - packet(Config, Data, send, passive_recv_packet, ?MANY, raw, false). + packet(Config, Data, send, passive_raw, ?MANY, raw, false). %%-------------------------------------------------------------------- @@ -193,14 +199,14 @@ packet_raw_passive_some_big() -> packet_raw_passive_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send, passive_recv_packet, ?SOME, raw, false). + packet(Config, Data, send, passive_raw, ?SOME, raw, false). %%-------------------------------------------------------------------- packet_0_passive_many_small() -> [{doc,"Test packet option {packet, 0} in passive mode."}]. packet_0_passive_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 0}, equivalent to packet raw.", - packet(Config, Data, send, passive_recv_packet, ?MANY, 0, false). + packet(Config, Data, send, passive_raw, ?MANY, 0, false). %%-------------------------------------------------------------------- packet_0_passive_some_big() -> @@ -208,7 +214,7 @@ packet_0_passive_some_big() -> packet_0_passive_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send, passive_recv_packet, ?SOME, 0, false). + packet(Config, Data, send, passive_raw, ?SOME, 0, false). %%-------------------------------------------------------------------- packet_1_passive_many_small() -> @@ -296,7 +302,7 @@ packet_1_active_once_many_small() -> packet_1_active_once_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 1}", - packet(Config, Data, send_raw, active_once_raw, ?MANY, 1, once). + packet(Config, Data, send, active_once_packet, ?MANY, 1, once). %%-------------------------------------------------------------------- packet_1_active_once_some_big() -> @@ -304,7 +310,7 @@ packet_1_active_once_some_big() -> packet_1_active_once_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(255, "1")), - packet(Config, Data, send_raw, active_once_raw, ?SOME, 1, once). + packet(Config, Data, send, active_once_packet, ?SOME, 1, once). %%-------------------------------------------------------------------- @@ -313,7 +319,7 @@ packet_2_active_once_many_small() -> packet_2_active_once_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 2}", - packet(Config, Data, send_raw, active_once_raw, ?MANY, 2, once). + packet(Config, Data, send, active_once_packet, ?MANY, 2, once). %%-------------------------------------------------------------------- packet_2_active_once_some_big() -> @@ -321,7 +327,7 @@ packet_2_active_once_some_big() -> packet_2_active_once_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send_raw, active_once_raw, ?SOME, 2, once). + packet(Config, Data, send, active_once_raw, ?SOME, 2, once). %%-------------------------------------------------------------------- packet_4_active_once_many_small() -> @@ -329,7 +335,7 @@ packet_4_active_once_many_small() -> packet_4_active_once_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 4}", - packet(Config, Data, send_raw, active_once_raw, ?MANY, 4, once). + packet(Config, Data, send, active_once_packet, ?MANY, 4, once). %%-------------------------------------------------------------------- packet_4_active_once_some_big() -> @@ -337,7 +343,7 @@ packet_4_active_once_some_big() -> packet_4_active_once_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send_raw, active_once_raw, ?SOME, 4, once). + packet(Config, Data, send, active_once_packet, ?SOME, 4, once). %%-------------------------------------------------------------------- packet_raw_active_many_small() -> @@ -345,7 +351,7 @@ packet_raw_active_many_small() -> packet_raw_active_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, raw}", - packet(Config, Data, send_raw, active_raw, ?MANY, raw, active). + packet(Config, Data, send_raw, active_raw, ?MANY, raw, true). %%-------------------------------------------------------------------- packet_raw_active_some_big() -> @@ -353,7 +359,7 @@ packet_raw_active_some_big() -> packet_raw_active_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send_raw, active_raw, ?SOME, raw, active). + packet(Config, Data, send_raw, active_raw, ?SOME, raw, true). %%-------------------------------------------------------------------- packet_0_active_many_small() -> @@ -361,7 +367,7 @@ packet_0_active_many_small() -> packet_0_active_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 0}", - packet(Config, Data, send_raw, active_raw, ?MANY, 0, active). + packet(Config, Data, send_raw, active_raw, ?MANY, 0, true). %%-------------------------------------------------------------------- packet_0_active_some_big() -> @@ -369,7 +375,7 @@ packet_0_active_some_big() -> packet_0_active_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send_raw, active_raw, ?SOME, 0, active). + packet(Config, Data, send, active_raw, ?SOME, 0, true). %%-------------------------------------------------------------------- packet_1_active_many_small() -> @@ -377,7 +383,7 @@ packet_1_active_many_small() -> packet_1_active_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 1}", - packet(Config, Data, send_raw, active_raw, ?MANY, 1, active). + packet(Config, Data, send, active_packet, ?MANY, 1, true). %%-------------------------------------------------------------------- packet_1_active_some_big() -> @@ -385,7 +391,7 @@ packet_1_active_some_big() -> packet_1_active_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(255, "1")), - packet(Config, Data, send_raw, active_raw, ?SOME, 1, active). + packet(Config, Data, send, active_packet, ?SOME, 1, true). %%-------------------------------------------------------------------- packet_2_active_many_small() -> @@ -393,7 +399,7 @@ packet_2_active_many_small() -> packet_2_active_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 2}", - packet(Config, Data, send_raw, active_raw, ?MANY, 2, active). + packet(Config, Data, send, active_packet, ?MANY, 2, true). %%-------------------------------------------------------------------- packet_2_active_some_big() -> @@ -401,7 +407,7 @@ packet_2_active_some_big() -> packet_2_active_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send_raw, active_raw, ?SOME, 2, active). + packet(Config, Data, send, active_packet, ?SOME, 2, true). %%-------------------------------------------------------------------- packet_4_active_many_small() -> @@ -409,7 +415,7 @@ packet_4_active_many_small() -> packet_4_active_many_small(Config) when is_list(Config) -> Data = "Packet option is {packet, 4}", - packet(Config, Data, send_raw, active_raw, ?MANY, 4, active). + packet(Config, Data, send, active_packet, ?MANY, 4, true). %%-------------------------------------------------------------------- packet_4_active_some_big() -> @@ -417,7 +423,7 @@ packet_4_active_some_big() -> packet_4_active_some_big(Config) when is_list(Config) -> Data = lists:append(lists:duplicate(100, "1234567890")), - packet(Config, Data, send_raw, active_raw, ?SOME, 4, active). + packet(Config, Data, send, active_packet, ?SOME, 4, true). %%-------------------------------------------------------------------- packet_send_to_large() -> @@ -1879,7 +1885,7 @@ packet(Config, Data, Send, Recv, Quantity, Packet, Active) -> Server = ssl_test_lib:start_server([{node, ClientNode}, {port, 0}, {from, self()}, {mfa, {?MODULE, Send ,[Data, Quantity]}}, - {options, ServerOpts}]), + {options, [{packet, Packet} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ServerNode}, {port, Port}, {host, Hostname}, diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 8d96a70a6e..76b302b1cb 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -203,67 +203,6 @@ close(Pid) -> ct:print("Pid: ~p down due to:~p ~n", [Pid, Reason]) end. - -check_result(Server, {error, SReason} = ServerMsg, Client, {error, closed} = ClientMsg) -> - receive - {Server, {error, {SReason, _}}} -> - receive - {Client, ClientMsg} -> - ok; - Unexpected -> - Reason = {{expected, {Client, ClientMsg}}, - {got, Unexpected}}, - ct:fail(Reason) - end; - {Client, ClientMsg} -> - receive - {Server, {error, {SReason, _}}} -> - ok; - Unexpected -> - Reason = {{expected, {Server,{error, {SReason, 'term()'}}}, - {got, Unexpected}}}, - ct:fail(Reason) - end; - {Port, {data,Debug}} when is_port(Port) -> - io:format("openssl ~s~n",[Debug]), - check_result(Server, ServerMsg, Client, ClientMsg); - - Unexpected -> - Reason = {{expected, {Client, ClientMsg}}, - {expected, {Server, {error, {SReason, 'term()'}}}, {got, Unexpected}}}, - ct:fail(Reason) - end; - -check_result(Server, {error, closed} = ServerMsg, Client, {error, CReson} = ClientMsg) -> - receive - {Server, ServerMsg} -> - receive - {Client, {error, {CReson, _}}} -> - ok; - Unexpected -> - Reason = {{expected, {Client, {error, {CReson, 'term()'}}}, - {got, Unexpected}}}, - ct:fail(Reason) - end; - {Client, {error, {CReson, _}}} -> - receive - {Server, ServerMsg} -> - ok; - Unexpected -> - Reason = {{expected, {Server, ServerMsg}}, - {got, Unexpected}}, - ct:fail(Reason) - end; - {Port, {data,Debug}} when is_port(Port) -> - io:format("openssl ~s~n",[Debug]), - check_result(Server, ServerMsg, Client, ClientMsg); - - Unexpected -> - Reason = {{expected, {Client, {error, {CReson, 'term()'}}}, - {expected, {Server, ServerMsg}}, {got, Unexpected}}}, - ct:fail(Reason) - end; - check_result(Server, ServerMsg, Client, ClientMsg) -> receive {Server, ServerMsg} -> @@ -294,22 +233,6 @@ check_result(Server, ServerMsg, Client, ClientMsg) -> ct:fail(Reason) end. -check_result(Pid, {error, Reason} = Err) when Reason == ecertfile; - Reason == ecacertfile; - Reason == ekeyfile; - Reason == edhfile -> - receive - {Pid, {error, {Reason, Str}}} when is_list(Str) -> - ok; - {Port, {data,Debug}} when is_port(Port) -> - io:format("openssl ~s~n",[Debug]), - check_result(Pid, Err); - Unexpected -> - Reason = {{expected, {Pid, {error, {Reason, "'appropriate error string'"}}}}, - {got, Unexpected}}, - ct:fail(Reason) - end; - check_result(Pid, Msg) -> receive {Pid, Msg} -> diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 7c0c00bf36..4f53132d5d 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -902,7 +902,7 @@ ssl2_erlang_server_openssl_client(Config) when is_list(Config) -> ok end, - ssl_test_lib:check_result(Server, {error, {essl, "protocol version"}}), + ssl_test_lib:check_result(Server, {error, {tls_alert, "protocol version"}}), process_flag(trap_exit, false). %%-------------------------------------------------------------------- diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index cb73e86ede..1f3bef83c8 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 5.2 +SSL_VSN = 5.2.1 diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 44c050a0d3..6db1ae3aa3 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2012</year> + <year>1996</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/stdlib/doc/src/filelib.xml b/lib/stdlib/doc/src/filelib.xml index ac67596f3c..bd780b2b2f 100644 --- a/lib/stdlib/doc/src/filelib.xml +++ b/lib/stdlib/doc/src/filelib.xml @@ -36,8 +36,9 @@ <description> <p>This module contains utilities on a higher level than the <c>file</c> module.</p> - <p>The module supports Unicode file names, so that it will match against regular expressions given in Unicode and that it will find and process raw file names (i.e. files named in a way that does not confirm to the expected encoding).</p> - <p>If the VM operates in Unicode file naming mode on a machine with transparent file naming, the <c>fun()</c> provided to <c>fold_files/5</c> needs to be prepared to handle binary file names.</p> + <p>This module does not support "raw" file names (i.e. files whose names + do not comply with the expected encoding). Such files will be ignored + by the functions in this module.</p> <p>For more information about raw file names, see the <seealso marker="kernel:file">file</seealso> module.</p> </description> diff --git a/lib/stdlib/doc/src/io.xml b/lib/stdlib/doc/src/io.xml index 63f814ad2e..90f24c4cbc 100644 --- a/lib/stdlib/doc/src/io.xml +++ b/lib/stdlib/doc/src/io.xml @@ -211,6 +211,18 @@ </desc> </func> <func> + <name name="printable_range" arity="0"/> + <fsummary>Get user requested printable character range</fsummary> + <desc> + <p>Return the user requested range of printable Unicode characters.</p> + <p>The user can request a range of characters that are to be considered printable in heuristic detection of strings by the shell and by the formatting functions. This is done by supplying <c>+pc <range></c> when starting Erlang.</p> + <p>Currently the only valid values for <c><range></c> are <c>latin1</c> and <c>unicode</c>. <c>latin1</c> means that only code points below 256 (with the exception of control characters etc) will be considered printable. <c>unicode</c> means that all printable characters in all unicode character ranges are considered printable by the io functions.</p> + <p>By default, Erlang is started so that only the <c>latin1</c> range of characters will indicate that a list of integers is a string.</p> + <p>The simplest way to utilize the setting is to call <seealso marker="io_lib#printable_list/1">io_lib:printable_list/1</seealso>, which will use the return value of this function to decide if a list is a string of printable characters or not.</p> + <note><p>In the future, this function may return more values and ranges. It is recommended to use the io_lib:printable_list/1 function to avoid compatibility problems.</p></note> + </desc> + </func> + <func> <name name="setopts" arity="1"/> <name name="setopts" arity="2"/> <fsummary>Set options</fsummary> diff --git a/lib/stdlib/doc/src/io_lib.xml b/lib/stdlib/doc/src/io_lib.xml index 001d34a7c2..3dac259477 100644 --- a/lib/stdlib/doc/src/io_lib.xml +++ b/lib/stdlib/doc/src/io_lib.xml @@ -301,7 +301,11 @@ <fsummary>Test for a list of printable characters</fsummary> <desc> <p>Returns <c>true</c> if <c><anno>Term</anno></c> is a flat list of - printable Unicode characters, otherwise it returns <c>false</c>.</p> + printable characters, otherwise it returns <c>false</c>.</p> + <p>What is a printable character in this case is determined by the + <c>+pc</c> start up flag to the Erlang VM. See + <seealso marker="io#printable_range/0">io:printable_range/0</seealso> + and <seealso marker="erts:erl#erl">erl(1)</seealso>.</p> </desc> </func> <func> @@ -312,6 +316,14 @@ printable ISO-latin-1 characters, otherwise it returns <c>false</c>.</p> </desc> </func> + <func> + <name name="printable_unicode_list" arity="1"/> + <fsummary>Test for a list of printable Unicode characters</fsummary> + <desc> + <p>Returns <c>true</c> if <c><anno>Term</anno></c> is a flat list of + printable Unicode characters, otherwise it returns <c>false</c>.</p> + </desc> + </func> </funcs> </erlref> diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index 13575588fd..2ec0d6a60f 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,75 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 1.19.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Bugs related to Unicode have been fixed in the + <c>erl_eval</c> module. </p> + <p> + Own Id: OTP-10622 Aux Id: kunagi-351 [262] </p> + </item> + <item> + <p><c>filelib:wildcard("some/relative/path/*.beam", + Path)</c> would fail to match any file. That is, + filelib:wildcard/2 would not work if the first component + of the pattern did not contain any wildcard characters. + (A previous attempt to fix the problem in R15B02 seems to + have made matters worse.)</p> + <p>(Thanks to Samuel Rivas and Tuncer Ayaz.)</p> + <p>There is also an incompatible change to the + <c>Path</c> argument. It is no longer allowed to be a + binary.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-10812</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> The new STDLIB application variable + <c>shell_strings</c> can be used for determining how the + Erlang shell outputs lists of integers. The new function + <c>shell:strings/1</c> toggles the value of the variable. + </p> <p> The control sequence modifier <c>l</c> can be + used for turning off the string recognition of <c>~p</c> + and <c>~P</c>. </p> + <p> + Own Id: OTP-10755</p> + </item> + <item> + <p> Miscellaneous updates due to Unicode support. </p> + <p> + Own Id: OTP-10820</p> + </item> + <item> + <p> Extend <c>~ts</c> to handle binaries with characters + coded in ISO-latin-1 </p> + <p> + Own Id: OTP-10836</p> + </item> + <item> + <p> + The +pc flag to erl can be used to set the range of + characters considered printable. This affects how the + shell and io:format("~tp",...) functionality does + heuristic string detection. More can be read in STDLIB + users guide.</p> + <p> + Own Id: OTP-10884</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 1.19</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml index 354ec58df3..c5d476e54b 100644 --- a/lib/stdlib/doc/src/unicode_usage.xml +++ b/lib/stdlib/doc/src/unicode_usage.xml @@ -32,50 +32,338 @@ <rev>PA1</rev> <file>unicode_usage.xml</file> </header> -<p>Implementing support for Unicode character sets is an ongoing process. The Erlang Enhancement Proposal (EEP) 10 outlines the basics of Unicode support and also specifies a default encoding in binaries that all Unicode-aware modules should handle in the future.</p> -<p>The functionality described in EEP10 is implemented in Erlang/OTP as of R13A, but that is by no means the end of it. More functionality will be needed in the future and more OTP-libraries might need updating to cope with Unicode data.</p> -<p>This guide outlines the current Unicode support and gives a couple of recipes for working with Unicode data.</p> +<section> +<title>Unicode Implementation</title> + <p>Implementing support for Unicode character sets is an ongoing + process. The Erlang Enhancement Proposal (EEP) 10 outlined the + basics of Unicode support and also specified a default encoding in + binaries that all Unicode-aware modules should handle in the + future.</p> + + <p>The functionality described in EEP10 was implemented in Erlang/OTP + as of R13A, but that was by no means the end of it. In R14B01 support + for Unicode file names was added, although it was in no way complete + and was by default disabled on platforms where no guarantee was given + for the file name encoding. With R16A came support for UTF-8 encoded + source code, among with enhancements to many of the applications to + support both Unicode encoded file names as well as support for UTF-8 + encoded files in several circumstances. Most notable is the support + for UTF-8 in files read by <c>file:consult/1</c>, release handler support + for UTF-8 and more support for Unicode character sets in the + I/O-system.</p> + + <p>In R17, the encoding default for Erlang source files will be + switched to UTF-8 and in R18 Erlang will support atoms in the full + Unicode range, meaning full Unicode function and module + names</p> + + <p>This guide outlines the current Unicode support and gives a couple + of recipes for working with Unicode data.</p> +</section> +<section> +<title>Understanding Unicode</title> + <p>Experience with the Unicode support in Erlang has made it + painfully clear that understanding Unicode characters and encodings + is not as easy as one would expect. The complexity of the field as + well as the implications of the standard requires thorough + understanding of concepts rarely before thought of.</p> + + <p>Furthermore the Erlang implementation requires understanding of + concepts that never were an issue for many (Erlang) programmers. To + understand and use Unicode characters requires that you study the + subject thoroughly, even if you're an experienced programmer.</p> + + <p>As an example, one could contemplate the issue of converting + between upper and lower case letters. Reading the standard will make + you realize that, to begin with, there's not a simple one to one + mapping in all scripts. Take German as an example, where there's a + letter "ß" (Sharp s) in lower case, but the uppercase equivalent is + "SS". Or Greek, where "Σ" has two different lowercase forms: "ς" in + word-final position and "σ" elsewhere. Or Turkish where dotted and + dot-less "i" both exist in lower case and upper case forms, or + Cyrillic "I" which usually has no lowercase form. Or of course + languages that have no concept of upper case (or lower case). So, a + conversion function will need to know not only one character at a + time, but possibly the whole sentence, maybe the natural language + the translation should be in and also take into account differences + in input and output string length and so on. There is at the time of + writing no Unicode to_upper/to_lower functionality in Erlang/OTP, but + there are publicly available libraries that address these issues.</p> + + <p>Another example is the accented characters where the same glyph + has two different representations. Let's look at the Swedish + "ö". There's a code point for that in the Unicode standard, but you + can also write it as "o" followed by U+0308 (Combining Diaeresis, + with the simplified meaning that the last letter should have a "¨" + above). They have exactly the same glyph. They are for most + purposes the same, but they have completely different + representations. For example MacOS X converts all file names to use + Combining Diaeresis, while most other programs (including Erlang) + try to hide that by doing the opposite when for example listing + directories. However it's done, it's usually important to normalize + such characters to avoid utter confusion.</p> + + <p>The list of examples can be made as long as the Unicode standard, I + suspect. The point is that one need a kind of knowledge that was + never needed when programs only took one or two languages into + account. The complexity of human languages and scripts, certainly + has made this a challenge when constructing a universal + standard. Supporting Unicode properly in your program <em>will</em> require + effort.</p> + +</section> <section> <title>What Unicode Is</title> -<p>Unicode is a standard defining codepoints (numbers) for all known, living or dead, scripts. In principle, every known symbol used in any language has a Unicode codepoint.</p> -<p>Unicode codepoints are defined and published by the <em>Unicode Consortium</em>, which is a non profit organization.</p> -<p>Support for Unicode is increasing throughout the world of computing, as the benefits of one common character set are overwhelming when programs are used in a global environment.</p> -<p>Along with the base of the standard, the codepoints for all the scripts, there are a couple of encoding standards available. Different operating systems and tools support different encodings. For example Linux and MacOSX has chosen the UTF-8 encoding, which is backwards compatible with 7-bit ASCII and therefore affects programs written in plain English the least. Windows® on the other hand supports a limited version of UTF-16, namely all the code planes where the characters can be stored in one single 16-bit entity, which includes most living languages.</p> -<p>The most widely spread encodings are:</p> -<taglist> -<tag>UTF-8</tag> -<item>Each character is stored in one to four bytes depending on codepoint. The encoding is backwards compatible with 7-bit ASCII as all 7-bit characters are stored in one single byte as is. The characters beyond codepoint 127 are stored in more bytes, letting the most significant bit in the first character indicate a multi-byte character. For details on the encoding, the RFC is publicly available.</item> -<tag>UTF-16</tag> -<item>This encoding has many similarities to UTF-8, but the basic unit is a 16-bit number. This means that all characters occupy at least two bytes, some high numbers even four bytes. Some programs and operating systems claiming to use UTF-16 only allows for characters that can be stored in one 16-bit entity, which is usually sufficient to handle living languages. As the basic unit is more than one byte, byte-order issues occur, why UTF-16 exists in both a big-endian and little-endian variant.</item> -<tag>UTF-32</tag> -<item>The most straight forward representation, each character is stored in one single 32-bit number. There is no need for escapes or any variable amount of entities for one character, all Unicode codepoints can be stored in one single 32-bit entity. As with UTF-16, there are byte-order issues, UTF-32 can be both big- and little-endian.</item> -<tag>UCS-4</tag> -<item>Basically the same as UTF-32, but without some Unicode semantics, defined by IEEE and has little use as a separate encoding standard. For all normal (and possibly abnormal) usages, UTF-32 and UCS-4 are interchangeable.</item> -</taglist> -<p>Certain ranges of characters are left unused and certain ranges are even deemed invalid. The most notable invalid range is 16#D800 - 16#DFFF, as the UTF-16 encoding does not allow for encoding of these numbers. It can be speculated that the UTF-16 encoding standard was, from the beginning, expected to be able to hold all Unicode characters in one 16-bit entity, but then had to be extended, leaving a hole in the Unicode range to cope with backward compatibility.</p> -<p>Additionally, the codepoint 16#FEFF is used for byte order marks (BOM's) and use of that character is not encouraged in other contexts than that. It actually is valid though, as the character "ZWNBS" (Zero Width Non Breaking Space). BOM's are used to identify encodings and byte order for programs where such parameters are not known in advance. Byte order marks are more seldom used than one could expect, but their use is becoming more widely spread as they provide the means for programs to make educated guesses about the Unicode format of a certain file.</p> + <p>Unicode is a standard defining code points (numbers) for all + known, living or dead, scripts. In principle, every known symbol + used in any language has a Unicode code point.</p> + <p>Unicode code points are defined and published by the <em>Unicode + Consortium</em>, which is a non profit organization.</p> + <p>Support for Unicode is increasing throughout the world of + computing, as the benefits of one common character set are + overwhelming when programs are used in a global environment.</p> + <p>Along with the base of the standard: the code points for all the + scripts, there are a couple of <em>encoding standards</em> available.</p> + <p>It is vital to understand the difference between encodings and + Unicode characters. Unicode characters are code points according to + the Unicode standard, while the encodings are ways to represent such + code points. An encoding is just a standard for representation, + UTF-8 can for example be used to represent a very limited part of + the Unicode character set (e.g. ISO-Latin-1), or the full Unicode + range. It's just an encoding format.</p> + <p>As long as all character sets were limited to 256 characters, + each character could be stored in one single byte, so there was more + or less only one practical encoding for the characters. Encoding + each character in one byte was so common that the encoding wasn't + even named. When we now, with the Unicode system, have a lot more + than 256 characters, we need a common way to represent these. The + common ways of representing the code points are the encodings. This + means a whole new concept to the programmer, the concept of + character representation, which was before a non-issue.</p> + + <p>Different operating systems and tools support different + encodings. For example Linux and MacOS X has chosen the UTF-8 + encoding, which is backwards compatible with 7-bit ASCII and + therefore affects programs written in plain English the + least. Windows on the other hand supports a limited version of + UTF-16, namely all the code planes where the characters can be + stored in one single 16-bit entity, which includes most living + languages.</p> + + <p>The most widely spread encodings are:</p> + <taglist> + <tag>Bytewise representation</tag> + <item>This is not a proper Unicode representation, but the + representation used for characters before the Unicode standard. It + can still be used to represent character code points in the Unicode + standard that have numbers below 256, which corresponds exactly to + the ISO-Latin-1 character set. In Erlang, this is commonly denoted + <c>latin1</c> encoding, which is slightly misleading as ISO-Latin-1 is + a character code range, not an encoding.</item> + <tag>UTF-8</tag> + <item>Each character is stored in one to four bytes depending on + code point. The encoding is backwards compatible with bytewise + representation of 7-bit ASCII as all 7-bit characters are stored + in one single byte in UTF-8. The characters beyond code point 127 + are stored in more bytes, letting the most significant bit in the + first character indicate a multi-byte character. For details on + the encoding, the RFC is publicly available. Note that UTF-8 is + <em>not</em> compatible with bytewise representation for + code points between 128 and 255, so a ISO-Latin-1 bytewise + representation is not generally compatible with UTF-8.</item> + <tag>UTF-16</tag> + <item>This encoding has many similarities to UTF-8, but the basic + unit is a 16-bit number. This means that all characters occupy at + least two bytes, some high numbers even four bytes. Some programs, + libraries and operating systems claiming to use UTF-16 only allows + for characters that can be stored in one 16-bit entity, which is + usually sufficient to handle living languages. As the basic unit + is more than one byte, byte-order issues occur, why UTF-16 exists + in both a big-endian and little-endian variant. In Erlang, the + full UTF-16 range is supported when applicable, like in the + <c>unicode</c> module and in the bit syntax.</item> + <tag>UTF-32</tag> + <item>The most straight forward representation. Each character is + stored in one single 32-bit number. There is no need for escapes + or any variable amount of entities for one character, all Unicode + code points can be stored in one single 32-bit entity. As with + UTF-16, there are byte-order issues, UTF-32 can be both big- and + little-endian.</item> + <tag>UCS-4</tag> + <item>Basically the same as UTF-32, but without some Unicode + semantics, defined by IEEE and has little use as a separate + encoding standard. For all normal (and possibly abnormal) usages, + UTF-32 and UCS-4 are interchangeable.</item> + </taglist> + <p>Certain ranges of numbers are left unused in the Unicode standard + and certain ranges are even deemed invalid. The most notable invalid + range is 16#D800 - 16#DFFF, as the UTF-16 encoding does not allow + for encoding of these numbers. It can be speculated that the UTF-16 + encoding standard was, from the beginning, expected to be able to + hold all Unicode characters in one 16-bit entity, but then had to be + extended, leaving a hole in the Unicode range to cope with backward + compatibility.</p> + <p>Additionally, the code point 16#FEFF is used for byte order marks + (BOM's) and use of that character is not encouraged in other + contexts than that. It actually is valid though, as the character + "ZWNBS" (Zero Width Non Breaking Space). BOM's are used to identify + encodings and byte order for programs where such parameters are not + known in advance. Byte order marks are more seldom used than one + could expect, but their use might become more widely spread as they + provide the means for programs to make educated guesses about the + Unicode format of a certain file.</p> </section> <section> -<title>Standard Unicode Representation in Erlang</title> -<p>In Erlang, strings are actually lists of integers. A string is defined to be encoded in the ISO-latin-1 (ISO8859-1) character set, which is, codepoint by codepoint, a sub-range of the Unicode character set.</p> -<p>The standard list encoding for strings is therefore easily extendible to cope with the whole Unicode range: A Unicode string in Erlang is simply a list containing integers, each integer being a valid Unicode codepoint and representing one character in the Unicode character set.</p> -<p>Regular Erlang strings in ISO-latin-1 are a subset of their Unicode -strings.</p> - -<p>Binaries on the other hand are more troublesome. For performance reasons, programs often store textual data in binaries instead of lists, mainly because they are more compact (one byte per character instead of two words per character, as is the case with lists). Using <c>erlang:list_to_binary/1</c>, a regular Erlang string can be converted into a binary, effectively using the ISO-latin-1 encoding in the binary - one byte per character. This is very convenient for those regular Erlang strings, but cannot be done for Unicode lists.</p> -<p>As the UTF-8 encoding is widely spread and provides the most compact storage, it is selected as the standard encoding of Unicode characters in binaries for Erlang.</p> -<p>The standard binary encoding is used whenever a library function in Erlang should cope with Unicode data in binaries, but is of course not enforced when communicating externally. Functions and bit-syntax exist to encode and decode both UTF-8, UTF-16 and UTF-32 in binaries. Library functions dealing with binaries and Unicode in general, however, only deal with the default encoding.</p> - -<p>Character data may be combined from several sources, sometimes available in a mix of strings and binaries. Erlang has for long had the concept of <c>iodata</c> or <c>iolists</c>, where binaries and lists can be combined to represent a sequence of bytes. In the same way, the Unicode aware modules often allow for combinations of binaries and lists where the binaries have characters encoded in UTF-8 and the lists contain such binaries or numbers representing Unicode codepoints:</p> -<code type="none"> + <title>Areas of Unicode Support</title> + <p>To support Unicode in Erlang, problems in several areas have been + addressed. Each area is described briefly in this section and more + thoroughly further down in this document:</p> + <taglist> + <tag>Representation</tag> + <item>To handle Unicode characters in Erlang, we have to have a + common representation both in lists and binaries. The EEP (10) and + the subsequent initial implementation in R13A settled a standard + representation of Unicode characters in Erlang.</item> + <tag>Manipulation</tag> + <item>The Unicode characters need to be processed by the Erlang + program, why library functions need to be able to handle them. In + some cases functionality was added to already existing interfaces + (as the string module now can handle lists with arbitrary code points), + in some cases new functionality or options need to be added (as in + the <c>io</c>-module, the file handling, the <c>unicode</c> module + and the bit syntax). Today most modules in kernel and STDLIB, as + well as the VM are Unicode aware.</item> + <tag>File I/O</tag> + <item>I/O is by far the most problematic area for Unicode. A file + is an entity where bytes are stored and the lore of programming + has been to treat characters and bytes as interchangeable. With + Unicode characters, you need to decide on an encoding as soon as + you want to store the data in a file. In Erlang you can open a + text file with an encoding option, so that you can read characters + from it rather than bytes, but you can also open a file for + bytewise I/O. The I/O-system of Erlang has been designed (or at + least used) in a way where you expect any I/O-server to be + able to cope with any string data, but that is no longer the case + when you work with Unicode characters. Handling the fact that you + need to know the capabilities of the device where your data ends + up is something new to the Erlang programmer. Furthermore, ports + in Erlang are byte oriented, so an arbitrary string of (Unicode) + characters can not be sent to a port without first converting it + to an encoding of choice.</item> + <tag>Terminal I/O</tag> + <item>Terminal I/O is slightly easier than file I/O. The output is + meant for human reading and is usually Erlang syntax (e.g. in the + shell). There exists syntactic representation of any Unicode + character without actually displaying the glyph (instead written + as <c>\x{</c>HHH<c>}</c>), so Unicode data can usually be displayed + even if the terminal as such do not support the whole Unicode + range.</item> + <tag>File names</tag> + <item>File names can be stored as Unicode strings, in different + ways depending on the underlying OS and file system. This can be + handled fairly easy by a program. The problems arise when the file + system is not consistent in it's encodings, like for example + Linux. Linux allows files to be named with any sequence of bytes, + leaving to each program to interpret those bytes. On systems where + these "transparent" file names are used, Erlang has to be informed + about the file name encoding by a startup flag. The default is + bytewise interpretation, which is actually usually wrong, but + allows for interpretation of <em>all</em> file names. The concept + of "raw file names" can be used to handle wrongly encoded + file names if one enables Unicode file name translation + (<c>+fnu</c>) on platforms where this is not the default.</item> + <tag>Source code encoding</tag> + <item>When it comes to the Erlang source code, there is support + for the UTF-8 encoding and bytewise encoding. The default in R16B + is bytewise (or latin1) encoding. You can control the encoding by + a comment like: +<code> +%% -*- coding: utf-8 -*- +</code> + in the beginning of the file. This of course requires your editor to + support UTF-8 as well. The same comment is also interpreted by + functions like <c>file:consult/1</c>, the release handler etc, so that + you can have all text files in your source directories in UTF-8 + encoding. + </item> + <tag>The language</tag> + <item>Having the source code in UTF-8 also allows you to write + string literals containing Unicode characters with code points > + 255, although atoms, module names and function names will be + restricted to the ISO-Latin-1 range until the R18 release. Binary + literals where you use the <c>/utf8</c> type, can also be + expressed using Unicode characters > 255. Having module names + using characters other than 7-bit ASCII can cause trouble on + operating systems with inconsistent file naming schemes, and might + also hurt portability, so it's not really recommended. It is + suggested in EEP 40 that the language should also allow for + Unicode characters > 255 in variable names. Whether to + implement that EEP or not is yet to be decided.</item> + </taglist> +</section> +<section> + <title>Standard Unicode Representation</title> + <p>In Erlang, strings are actually lists of integers. A string was + up until R13 defined to be encoded in the ISO-latin-1 (ISO8859-1) + character set, which is, code point by code point, a sub-range of + the Unicode character set.</p> + <p>The standard list encoding for strings was therefore easily + extended to cope with the whole Unicode range: A Unicode string in + Erlang is simply a list containing integers, each integer being a + valid Unicode code point and representing one character in the + Unicode character set.</p> + <p>Erlang strings in ISO-latin-1 are a subset of Unicode + strings.</p> + <p>Only if a string contains code points < 256, can it be + directly converted to a binary by using + i.e. <c>erlang:iolist_to_binary/1</c> or can be sent directly to a + port. If the string contains Unicode characters > 255, an + encoding has to be decided upon and the string should be converted + to a binary in the preferred encoding using + <c>unicode:characters_to_binary/{1,2,3}</c>. Strings are not + generally lists of bytes, as they were before R13. They are lists of + characters. Characters are not generally bytes, they are Unicode + code points.</p> + + <p>Binaries are more troublesome. For performance reasons, programs + often store textual data in binaries instead of lists, mainly + because they are more compact (one byte per character instead of two + words per character, as is the case with lists). Using + <c>erlang:list_to_binary/1</c>, an ISO-Latin-1 Erlang string could + be converted into a binary, effectively using bytewise encoding - + one byte per character. This was very convenient for those limited + Erlang strings, but cannot be done for arbitrary Unicode lists.</p> + <p>As the UTF-8 encoding is widely spread and provides some backward + compatibility in the 7-bit ASCII range, it is selected as the + standard encoding for Unicode characters in binaries for Erlang.</p> + <p>The standard binary encoding is used whenever a library function + in Erlang should cope with Unicode data in binaries, but is of + course not enforced when communicating externally. Functions and + bit-syntax exist to encode and decode both UTF-8, UTF-16 and UTF-32 + in binaries. Library functions dealing with binaries and Unicode in + general, however, only deal with the default encoding.</p> + + <p>Character data may be combined from several sources, sometimes + available in a mix of strings and binaries. Erlang has for long had + the concept of <c>iodata</c> or <c>iolist</c>s, where binaries and + lists can be combined to represent a sequence of bytes. In the same + way, the Unicode aware modules often allow for combinations of + binaries and lists where the binaries have characters encoded in + UTF-8 and the lists contain such binaries or numbers representing + Unicode code points:</p> + <code type="none"> unicode_binary() = binary() with characters encoded in UTF-8 coding standard chardata() = charlist() | unicode_binary() charlist() = maybe_improper_list(char() | unicode_binary() | charlist(), unicode_binary() | nil())</code> -<p>The module <c>unicode</c> in STDLIB even supports similar mixes with binaries containing other encodings than UTF-8, but that is a special case to allow for conversions to and from external data:</p> - <code type="none"> + <p>The module <seealso + marker="stdlib:unicode"><c>unicode</c></seealso> in STDLIB even + supports similar mixes with binaries containing other encodings than + UTF-8, but that is a special case to allow for conversions to and + from external data:</p> + <code type="none"> external_unicode_binary() = binary() with characters coded in a user specified Unicode encoding other than UTF-8 (UTF-16 or UTF-32) @@ -87,185 +375,741 @@ external_charlist() = maybe_improper_list(char() | external_unicode_binary() | nil())</code> </section> <section> -<title>Basic Language Support for Unicode</title> -<p><marker id="unicode_in_erlang"/>As of Erlang/OTP R16 Erlang can be -written in ISO-latin-1 or Unicode (UTF-8). The details on how to state -the encoding of an Erlang source file can be found in <seealso -marker="stdlib:epp#encoding">epp(3)</seealso>. Strings and comments -can be written using Unicode, but functions still have to be named in -ISO-latin-1 and atoms are restricted to ISO-latin-1. Erlang/OTP R18 is -expected to handle functions named in Unicode as well as Unicode -atoms.</p> -<section> -<title>Bit-syntax</title> -<p>The bit-syntax contains types for coping with binary data in the three main encodings. The types are named <c>utf8</c>, <c>utf16</c> and <c>utf32</c> respectively. The <c>utf16</c> and <c>utf32</c> types can be in a big- or little-endian variant:</p> -<code> + <title>Basic Language Support</title> + <p><marker id="unicode_in_erlang"/>As of Erlang/OTP R16 Erlang + source files can be written in either UTF-8 or bytewise encoding + (a.k.a. <c>latin1</c> encoding). The details on how to state the encoding + of an Erlang source file can be found in + <seealso marker="stdlib:epp#encoding"><c>epp(3)</c></seealso>. Strings and comments + can be written using Unicode, but functions still have to be named + using characters from the ISO-latin-1 character set and atoms are + restricted to the same ISO-latin-1 range. These restrictions in the + language are of course independent of the encoding of the source + file. Erlang/OTP R18 is expected to handle functions named in + Unicode as well as Unicode atoms.</p> + <section> + <title>Bit-syntax</title> + <p>The bit-syntax contains types for coping with binary data in the + three main encodings. The types are named <c>utf8</c>, <c>utf16</c> + and <c>utf32</c> respectively. The <c>utf16</c> and <c>utf32</c> types + can be in a big- or little-endian variant:</p> + <code> <<Ch/utf8,_/binary>> = Bin1, <<Ch/utf16-little,_/binary>> = Bin2, Bin3 = <<$H/utf32-little, $e/utf32-little, $l/utf32-little, $l/utf32-little, - $o/utf32-little>>,</code> -<p>For convenience, literal strings can be encoded with a Unicode encoding in binaries using the following (or similar) syntax:</p> -<code> +$o/utf32-little>>,</code> + <p>For convenience, literal strings can be encoded with a Unicode + encoding in binaries using the following (or similar) syntax:</p> + <code> Bin4 = <<"Hello"/utf16>>,</code> -</section> -<section> -<title>String- and Character-literals</title> -<p>For source code, there is an extension to the <c>\</c>OOO (backslash -followed by three octal numbers) and <c>\x</c>HH (backslash followed by <c>x</c>, -followed by two hexadecimal characters) syntax, namely <c>\x{</c>H ...<c>}</c> (a -backslash followed by an <c>x</c>, followed by left curly bracket, any -number of hexadecimal digits and a terminating right curly bracket). -This allows for entering characters of any codepoint literally in a -string even when the encoding is ISO-latin-1.</p> -</section> -<p>In the shell, if using a Unicode input device, <c>$</c> can be followed directly by a Unicode character producing an integer. In the following example the codepoint of a Cyrillic <c>s</c> is output:</p> -<pre> + </section> + <section> + <title>String and Character Literals</title> + <p>For source code, there is an extension to the <c>\</c>OOO + (backslash followed by three octal numbers) and <c>\x</c>HH + (backslash followed by <c>x</c>, followed by two hexadecimal + characters) syntax, namely <c>\x{</c>H ...<c>}</c> (a backslash + followed by an <c>x</c>, followed by left curly bracket, any + number of hexadecimal digits and a terminating right curly + bracket). This allows for entering characters of any code point + literally in a string even when the encoding of the source file is + bytewise (<c>latin1</c>).</p> + <p>In the shell, if using a Unicode input device, or in source + code stored in UTF-8, <c>$</c> can be followed directly by a + Unicode character producing an integer. In the following example + the code point of a Cyrillic <c>с</c> is output:</p> + <pre> 7> <input>$с.</input> 1089</pre> + </section> + <section> + <title>Heuristic String Detection</title> + <p>In certain output functions and in the output of return values + in the shell, Erlang tries to heuristically detect string data in + lists and binaries. Typically you will see heuristic detection in + a situation like this:</p> + <pre> +1> <input>[97,98,99].</input> +"abc" +2> <input><<97,98,99>>.</input> +<<"abc">> +3> <input><<195,165,195,164,195,182>>.</input> +<<"åäö"/utf8>></pre> + <p>Here the shell will detect lists containing printable + characters or binaries containing printable characters either in + bytewise or UTF-8 encoding. The question here is: what is a + printable character? One view would be that anything the Unicode + standard thinks is printable, will also be printable according to + the heuristic detection. The result would be that almost any list + of integers will be deemed a string, resulting in all sorts of + characters being printed, maybe even characters your terminal does + not have in its font set (resulting in some generic output you + probably will not appreciate). Another way is to keep it backwards + compatible so that only the ISO-Latin-1 character set is used to + detect a string. A third way would be to let the user decide + exactly what Unicode ranges are to be viewed as characters. In + R16B you can select either the whole Unicode range or the + ISO-Latin-1 range by supplying the startup flag <c>+pc + </c><i>Range</i>, where <i>Range</i> is either <c>latin1</c> or + <c>unicode</c>. For backwards compatibility, the default is + <c>latin1</c>. This only controls how heuristic string detection + is done. In the future, more ranges are expected to be added, so + that one can tailor the heuristics to the language and region + relevant to the user.</p> + <p>Lets look at an example with the two different startup options:</p> +<pre> +$ <input>erl +pc latin1</input> +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] + +Eshell V5.10.1 (abort with ^G) +1> <input>[1024].</input> +[1024] +2> <input>[1070,1085,1080,1082,1086,1076].</input> +[1070,1085,1080,1082,1086,1076] +3> <input>[229,228,246].</input> +"åäö" +4> <input><<208,174,208,189,208,184,208,186,208,190,208,180>>.</input> +<<208,174,208,189,208,184,208,186,208,190,208,180>> +5> <input><<229/utf8,228/utf8,246/utf8>>.</input> +<<"åäö"/utf8>> +</pre> +<pre> +$ <input>erl +pc unicode</input> +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] + +Eshell V5.10.1 (abort with ^G) +1> <input>[1024].</input> +"Ѐ" +2> <input>[1070,1085,1080,1082,1086,1076].</input> +"Юникод" +3> <input>[229,228,246].</input> +"åäö" +4> <input><<208,174,208,189,208,184,208,186,208,190,208,180>>.</input> +<<"Юникод"/utf8>> +5> <input><<229/utf8,228/utf8,246/utf8>>.</input> +<<"åäö"/utf8>> +</pre> + <p>In the examples, we can see that the default Erlang shell will + only interpret characters from the ISO-Latin1 range as printable + and will only detect lists or binaries with those "printable" + characters as containing string data. The valid UTF-8 binary + containing "Юникод", will not be printed as a string. When, on the + other hand, started with all Unicode characters printable (<c>+pc + unicode</c>), the shell will output anything containing printable + Unicode data (in binaries either UTF-8 or bytewise encoded) as + string data.</p> + + <p>These heuristics are also used by + <c>io</c>(<c>_lib</c>)<c>:format/2</c> and friends when the + <c>t</c> modifier is used in conjunction with <c>~p</c> or + <c>~P</c>:</p> +<pre> +$ <input>erl +pc latin1</input> +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] + +Eshell V5.10.1 (abort with ^G) +1> <input>io:format("~tp~n",[{<<"åäö">>, <<"åäö"/utf8>>, <<208,174,208,189,208,184,208,186,208,190,208,180>>}]).</input> +{<<"åäö">>,<<"åäö"/utf8>>,<<208,174,208,189,208,184,208,186,208,190,208,180>>} +ok +</pre> +<pre> +$ <input>erl +pc unicode</input> +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] + +Eshell V5.10.1 (abort with ^G) +1> <input>io:format("~tp~n",[{<<"åäö">>, <<"åäö"/utf8>>, <<208,174,208,189,208,184,208,186,208,190,208,180>>}]).</input> +{<<"åäö">>,<<"åäö"/utf8>>,<<"Юникод"/utf8>>} +ok +</pre> + <p>Please observe that this only affects <i>heuristic</i> interpretation + of lists and binaries on output. For example the <c>~ts</c> format + sequence does always output a valid lists of characters, + regardless of the <c>+pc</c> setting, as the programmer has + explicitly requested string output.</p> + </section> </section> <section> -<title>The Interactive Shell</title> -<p>The interactive Erlang shell, when started towards a terminal or started using the <c>werl</c> command on windows, can support Unicode input and output.</p> -<p>On Windows®, proper operation requires that a suitable font is installed and selected for the Erlang application to use. If no suitable font is available on your system, try installing the DejaVu fonts (<c>dejavu-fonts.org</c>), which are freely available and then select that font in the Erlang shell application.</p> -<p>On Unix®-like operating systems, the terminal should be able to handle UTF-8 on input and output (modern versions of XTerm, KDE konsole and the Gnome terminal do for example) and your locale settings have to be proper. As an example, my <c>LANG</c> environment variable is set as this:</p> -<pre> + <title>The Interactive Shell</title> + <p>The interactive Erlang shell, when started towards a terminal or + started using the <c>werl</c> command on windows, can support + Unicode input and output.</p> + <p>On Windows, proper operation requires that a suitable font + is installed and selected for the Erlang application to use. If no + suitable font is available on your system, try installing the DejaVu + fonts (<c>dejavu-fonts.org</c>), which are freely available and then + select that font in the Erlang shell application.</p> + <p>On Unix-like operating systems, the terminal should be able + to handle UTF-8 on input and output (modern versions of XTerm, KDE + konsole and the Gnome terminal do for example) and your locale + settings have to be proper. As an example, my <c>LANG</c> + environment variable is set as this:</p> + <pre> $ <input>echo $LANG</input> en_US.UTF-8</pre> -<p>Actually, most systems handle the <c>LC_CTYPE</c> variable before <c>LANG</c>, so if that is set, it has to be set to <c>UTF-8</c>:</p> -<pre> + <p>Actually, most systems handle the <c>LC_CTYPE</c> variable before + <c>LANG</c>, so if that is set, it has to be set to + <c>UTF-8</c>:</p> + <pre> $ echo <input>$LC_CTYPE</input> en_US.UTF-8</pre> -<p>The <c>LANG</c> or <c>LC_CTYPE</c> setting should be consistent with what the terminal is capable of, there is no portable way for Erlang to ask the actual terminal about its UTF-8 capacity, we have to rely on the language and character type settings.</p> -<p>To investigate what Erlang thinks about the terminal, the <c>io:getopts()</c> call can be used when the shell is started:</p> -<pre> + <p>The <c>LANG</c> or <c>LC_CTYPE</c> setting should be consistent + with what the terminal is capable of, there is no portable way for + Erlang to ask the actual terminal about its UTF-8 capacity, we have + to rely on the language and character type settings.</p> + <p>To investigate what Erlang thinks about the terminal, the + <c>io:getopts()</c> call can be used when the shell is started:</p> + <pre> $ <input>LC_CTYPE=en_US.ISO-8859-1 erl</input> -Erlang R16B (erts-5.10) [source] [async-threads:0] [hipe] [kernel-poll:false] +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] -Eshell V5.10 (abort with ^G) +Eshell V5.10.1 (abort with ^G) 1> <input>lists:keyfind(encoding, 1, io:getopts()).</input> {encoding,latin1} 2> <input>q().</input> ok $ <input>LC_CTYPE=en_US.UTF-8 erl</input> -Erlang R16B (erts-5.10) [source] [async-threads:0] [hipe] [kernel-poll:false] +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] -Eshell V5.10 (abort with ^G) +Eshell V5.10.1 (abort with ^G) 1> <input>lists:keyfind(encoding, 1, io:getopts()).</input> {encoding,unicode} 2></pre> -<p>When (finally?) everything is in order with the locale settings, fonts and the terminal emulator, you probably also have discovered a way to input characters in the script you desire. For testing, the simplest way is to add some keyboard mappings for other languages, usually done with some applet in your desktop environment. In my KDE environment, I start the KDE Control Center (Personal Settings), select "Regional and Accessibility" and then "Keyboard Layout". On Windows XP®, I start Control Panel->Regional and Language Options, select the Language tab and click the Details... button in the square named "Text services and input Languages". Your environment probably provides similar means of changing the keyboard layout. Make sure you have a way to easily switch back and forth between keyboards if you are not used to this, entering commands using a Cyrillic character set is, as an example, not easily done in the Erlang shell.</p> -<p>Now you are set up for some Unicode input and output. The simplest thing to do is of course to enter a string in the shell:</p> -<pre> + + <p>When (finally?) everything is in order with the locale settings, + fonts and the terminal emulator, you probably also have discovered a + way to input characters in the script you desire. For testing, the + simplest way is to add some keyboard mappings for other languages, + usually done with some applet in your desktop environment. In my KDE + environment, I start the KDE Control Center (Personal Settings), + select "Regional and Accessibility" and then "Keyboard Layout". On + Windows XP, I start Control Panel->Regional and Language + Options, select the Language tab and click the Details... button in + the square named "Text services and input Languages". Your + environment probably provides similar means of changing the keyboard + layout. Make sure you have a way to easily switch back and forth + between keyboards if you are not used to this, entering commands + using a Cyrillic character set is, as an example, not easily done in + the Erlang shell.</p> + + <p>Now you are set up for some Unicode input and output. The + simplest thing to do is of course to enter a string in the + shell:</p> + + <pre> $ <input>erl</input> -Erlang R16B (erts-5.10) [source] [async-threads:0] [hipe] [kernel-poll:false] +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] -Eshell V5.10 (abort with ^G) +Eshell V5.10.1 (abort with ^G) 1> <input>lists:keyfind(encoding, 1, io:getopts()).</input> {encoding,unicode} -2> <input>"Юникод"</input> +2> <input>"Юникод".</input> "Юникод" 3> <input>io:format("~ts~n", [v(2)]).</input> Юникод ok 4> </pre> -<p>While strings can be input as Unicode characters, the language elements are still limited to the ISO-latin-1 character set. Only character constants and strings are allowed to be beyond that range:</p> -<pre> + <p>While strings can be input as Unicode characters, the language + elements are still limited to the ISO-latin-1 character set. Only + character constants and strings are allowed to be beyond that + range:</p> + <pre> $ <input>erl</input> -Erlang R16B (erts-5.10) [source] [async-threads:0] [hipe] [kernel-poll:false] +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] -Eshell V5.10 (abort with ^G) -1> <input>$ξ</input> +Eshell V5.10.1 (abort with ^G) +1> <input>$ξ.</input> 958 2> <input>Юникод.</input> * 1: illegal character 2> </pre> </section> <section> -<title>Unicode File Names</title> -<p>Most modern operating systems support Unicode file names in some way or another. There are several different ways to do this and Erlang by default treats the different approaches differently:</p> -<taglist> -<tag>Mandatory Unicode file naming</tag> -<item> -<p>Windows and, for most common uses, MacOSX enforces Unicode support for file names. All files created in the filesystem have names that can consistently be interpreted. In MacOSX, all file names are retrieved in UTF-8 encoding, while Windows has selected an approach where each system call handling file names has a special Unicode aware variant, giving much the same effect. There are no file names on these systems that are not Unicode file names, why the default behavior of the Erlang VM is to work in "Unicode file name translation mode", meaning that a file name can be given as a Unicode list and that will be automatically translated to the proper name encoding for the underlying operating and file system.</p> -<p>Doing i.e. a <c>file:list_dir/1</c> on one of these systems may return Unicode lists with codepoints beyond 255, depending on the content of the actual filesystem.</p> -<p>As the feature is fairly new, you may still stumble upon non core applications that cannot handle being provided with file names containing characters with codepoints larger than 255, but the core Erlang system should have no problems with Unicode file names.</p> -</item> -<tag>Transparent file naming</tag> -<item> -<p>Most Unix operating systems have adopted a simpler approach, namely that Unicode file naming is not enforced, but by convention. Those systems usually use UTF-8 encoding for Unicode file names, but do not enforce it. On such a system, a file name containing characters having codepoints between 128 and 255 may be named either as plain ISO-latin-1 or using UTF-8 encoding. As no consistency is enforced, the Erlang VM can do no consistent translation of all file names. If the VM would automatically select encoding based on heuristics, one could get unexpected behavior on these systems, therefore file names not being encoded in UTF-8 are returned as "raw file names" if Unicode file naming support is turned on.</p> -<p>A raw file name is not a list, but a binary. Many non core applications still do not handle file names given as binaries, why such raw names are avoided by default. This means that systems having implemented Unicode file naming through transparent file systems and an UTF-8 convention, do not by default have Unicode file naming turned on. Explicitly turning Unicode file name handling on for these types of systems is considered experimental.</p> -</item> -</taglist> -<p>The Unicode file naming support was introduced with OTP release R14B01. A VM operating in Unicode file mode can work with files having names in any language or character set (as long as it is supported by the underlying OS and file system). The Unicode character list is used to denote file or directory names and if the file system content is listed, you will also be able to get Unicode lists as return value. The support lies in the Kernel and STDLIB modules, why most applications (that does not explicitly require the file names to be in the ISO-latin-1 range) will benefit from the Unicode support without change.</p> + <title>Unicode File Names</title> + <p>Most modern operating systems support Unicode file names in some + way or another. There are several different ways to do this and + Erlang by default treats the different approaches differently:</p> + <taglist> + <tag>Mandatory Unicode file naming</tag> + <item> + <p>Windows and, for most common uses, MacOS X enforces Unicode + support for file names. All files created in the file system have + names that can consistently be interpreted. In MacOS X, all file + names are retrieved in UTF-8 encoding, while Windows has + selected an approach where each system call handling file names + has a special Unicode aware variant, giving much the same + effect. There are no file names on these systems that are not + Unicode file names, why the default behavior of the Erlang VM is + to work in "Unicode file name translation mode", + meaning that a file name can be given as a Unicode list and that + will be automatically translated to the proper name encoding for + the underlying operating and file system.</p> + <p>Doing i.e. a <c>file:list_dir/1</c> on one of these systems + may return Unicode lists with code points beyond 255, depending + on the content of the actual file system.</p> + <p>As the feature is fairly new, you may still stumble upon non + core applications that cannot handle being provided with file + names containing characters with code points larger than 255, but + the core Erlang system should have no problems with Unicode file + names.</p> + </item> + <tag>Transparent file naming</tag> + <item> + <p>Most Unix operating systems have adopted a simpler approach, + namely that Unicode file naming is not enforced, but by + convention. Those systems usually use UTF-8 encoding for Unicode + file names, but do not enforce it. On such a system, a file name + containing characters having code points between 128 and 255 may + be named either as plain ISO-latin-1 or using UTF-8 encoding. As + no consistency is enforced, the Erlang VM can do no consistent + translation of all file names. If the VM would automatically + select encoding based on heuristics, one could get unexpected + behavior on these systems. By default, Erlang starts in "latin1" + file name mode on such systems, meaning bytewise encoding in file + names. This allows for list representation of all file names in + the system, but, for example, a file named "Östersund.txt", will + appear in <c>file:list_dir/1</c> as either "Östersund.txt" (if + the file name was encoded in bytewise ISO-Latin-1 by the program + creating the file, or more probably as + <c>[195,150,115,116,101,114,115,117,110,100]</c>, which is a + list containing UTF-8 bytes - not what you would want... If you + on the other hand use Unicode file name translation on such a + system, non-UTF-8 file names will simply be ignored by functions + like <c>file:list_dir/1</c>. They can be retrieved with + <c>file:list_dir_all/1</c>, but wrongly encoded file names will + appear as "raw file names".</p> -<p>On Operating systems with mandatory Unicode file names, this means that you more easily conform to the file names of other (non Erlang) applications, and you can also process file names that, at least on Windows, were completely inaccessible (due to having names that could not be represented in ISO-latin-1). Also you will avoid creating incomprehensible file names on MacOSX as the vfs layer of the OS will accept all your file names as UTF-8 and will not rewrite them.</p> + </item> + </taglist> -<p>For most systems, turning on Unicode file name translation is no problem even if it uses transparent file naming. Very few systems have mixed file name encodings. A consistent UTF-8 named system will work perfectly in Unicode file name mode. It is still however considered experimental in R14B01. Unicode file name translation is turned on with the <c>+fnu</c> switch to the <c>erl</c> program. If the VM is started in Unicode file name translation mode, <c>file:native_name_encoding/0</c> will return the atom <c>utf8</c>.</p> + <p>The Unicode file naming support was introduced with OTP release + R14B01. A VM operating in Unicode file name translation mode can + work with files having names in any language or character set (as + long as it is supported by the underlying OS and file system). The + Unicode character list is used to denote file or directory names and + if the file system content is listed, you will also get + Unicode lists as return value. The support lies in the Kernel and + STDLIB modules, why most applications (that does not explicitly + require the file names to be in the ISO-latin-1 range) will benefit + from the Unicode support without change.</p> -<p>In Unicode file name mode, file names given to the BIF <c>open_port/2</c> with the option <c>{spawn_executable,...}</c> are also interpreted as Unicode. So is the parameter list given in the <c>args</c> option available when using <c>spawn_executable</c>. The UTF-8 translation of arguments can be avoided using binaries, see the discussion about raw file names below.</p> + <p>On operating systems with mandatory Unicode file names, this + means that you more easily conform to the file names of other (non + Erlang) applications, and you can also process file names that, at + least on Windows, were completely inaccessible (due to having names + that could not be represented in ISO-latin-1). Also you will avoid + creating incomprehensible file names on MacOS X as the vfs layer of + the OS will accept all your file names as UTF-8 and will not rewrite + them.</p> -<p>It is worth noting that the file <c>encoding</c> options given when opening a file has nothing to do with the file <em>name</em> encoding convention. You can very well open files containing UTF-8 but having file names in ISO-latin-1 or vice versa.</p> + <p>For most systems, turning on Unicode file name translation is no + problem even if it uses transparent file naming. Very few systems + have mixed file name encodings. A consistent UTF-8 named system will + work perfectly in Unicode file name mode. It was still however + considered experimental in R14B01 and is still not the default on + such systems. Unicode file name translation is turned on with the + <c>+fnu</c> switch to the On Linux, a VM started without explicitly + stating the file name translation mode will default to <c>latin1</c> + as the native file name encoding. On Windows and MacOS X, the + default behavior is that of Unicode file name translation, why the + <c>file:native_name_encoding/0</c> by default returns <c>utf8</c> on + those systems (the fact that Windows actually does not use UTF-8 on + the file system level can safely be ignored by the Erlang + programmer). The default behavior can, as stated before, be + changed using the <c>+fnu</c> or <c>+fnl</c> options to the VM, see + the <seealso marker="erts:erl"><c>erl</c></seealso> program. If the + VM is started in Unicode file name translation mode, + <c>file:native_name_encoding/0</c> will return the atom + <c>utf8</c>. The <c>+fnu</c> switch can be followed by <c>w</c>, + <c>i</c> or <c>e</c>, to control how wrongly encoded file names are + to be reported. <c>w</c> means that a warning is sent to the + <c>error_logger</c> whenever a wrongly encoded file name is + "skipped" in directory listings, <c>i</c> means that those wrongly + encoded file names are silently ignored and <c>e</c> means that the + API function will return an error whenever a wrongly encoded file + (or directory) name is encountered. <c>w</c> is the default. Note + that <c>file:read_link/1</c> will always return an error if the link + points to an invalid file name.</p> -<note><p>Erlang drivers and NIF shared objects still can not be named with names containing codepoints beyond 127. This is a known limitation to be removed in a future release. Erlang modules however can, but it is definitely not a good idea and is still considered experimental.</p></note> + <p>In Unicode file name mode, file names given to the BIF + <c>open_port/2</c> with the option <c>{spawn_executable,...}</c> are + also interpreted as Unicode. So is the parameter list given in the + <c>args</c> option available when using <c>spawn_executable</c>. The + UTF-8 translation of arguments can be avoided using binaries, see + the discussion about raw file names below.</p> + + <p>It is worth noting that the file <c>encoding</c> options given + when opening a file has nothing to do with the file <em>name</em> + encoding convention. You can very well open files containing data + encoded in UTF-8 but having file names in bytewise (<c>latin1</c>) encoding + or vice versa.</p> + + <note><p>Erlang drivers and NIF shared objects still can not be + named with names containing code points beyond 127. This is a known + limitation to be removed in a future release. Erlang modules however + can, but it is definitely not a good idea and is still considered + experimental.</p></note> <section> -<title>Notes About Raw File Names and Automatic File Name Conversion</title> + <title>Notes About Raw File Names</title> + + <p>Raw file names were introduced together with Unicode file name + support in erts-5.8.2 (OTP R14B01). The reason "raw file + names" was introduced in the system was to be able to + consistently represent file names given in different encodings on + the same system. Having the VM automatically translate a file name + that is not in UTF-8 to a list of Unicode characters might seem + practical, but this would open up for both duplicate file names and + other inconsistent behavior. Consider a directory containing a file + named "björn" in ISO-latin-1, while the Erlang VM is + operating in Unicode file name mode (and therefore expecting UTF-8 + file naming). The ISO-latin-1 name is not valid UTF-8 and one could + be tempted to think that automatic conversion in for example + <c>file:list_dir/1</c> is a good idea. But what would happen if we + later tried to open the file and have the name as a Unicode list + (magically converted from the ISO-latin-1 file name)? The VM will + convert the file name given to UTF-8, as this is the encoding + expected. Effectively this means trying to open the file named + <<"björn"/utf8>>. This file does not exist, + and even if it existed it would not be the same file as the one that + was listed. We could even create two files named "björn", + one named in the UTF-8 encoding and one not. If + <c>file:list_dir/1</c> would automatically convert the ISO-latin-1 + file name to a list, we would get two identical file names as the + result. To avoid this, we need to differentiate between file names + being properly encoded according to the Unicode file naming + convention (i.e. UTF-8) and file names being invalid under the + encoding. By the common <c>file:list_dir/1</c> function, the wrongly + encoded file names are simply ignored in Unicode file name + translation mode, but by the <c>file:list_dir_all/1</c> function, + the file names with invalid encoding are returned as "raw" + file names, i.e. as binaries.</p> + + <p>The Erlang <c>file</c> module accepts raw file names as + input. <c>open_port({spawn_executable, ...} ...)</c> also accepts + them. As mentioned earlier, the arguments given in the option list + to <c>open_port({spawn_executable, ...} ...)</c> undergo the same + conversion as the file names, meaning that the executable will be + provided with arguments in UTF-8 as well. This translation is + avoided consistently with how the file names are treated, by giving + the argument as a binary.</p> + + <p>To force Unicode file name translation mode on systems where this + is not the default was considered experimental in OTP R14B01 due to + the fact that the initial implementation did not ignore wrongly + encoded file names, so that raw file names could spread unexpectedly + throughout the system. Beginning with R16B, the wrongly encoded file + names are only retrieved by special functions + (e.g. <c>file:list_dir_all/1</c>), so the impact on existing code is + much lower, why it is now supported. Unicode file name translation + is expected to be default in future releases.</p> -<p>Raw file names is introduced together with Unicode file name support in erts-5.8.2 (OTP R14B01). The reason "raw file names" is introduced in the system is to be able to consistently represent file names given in different encodings on the same system. Having the VM automatically translate a file name that is not in UTF-8 to a list of Unicode characters might seem practical, but this would open up for both duplicate file names and other inconsistent behavior. Consider a directory containing a file named "björn" in ISO-latin-1, while the Erlang VM is operating in Unicode file name mode (and therefore expecting UTF-8 file naming). The ISO-latin-1 name is not valid UTF-8 and one could be tempted to think that automatic conversion in for example <c>file:list_dir/1</c> is a good idea. But what would happen if we later tried to open the file and have the name as a Unicode list (magically converted from the ISO-latin-1 file name)? The VM will convert the file name given to UTF-8, as this is the encoding expected. Effectively this means trying to open the file named <<"björn"/utf8>>. This file does not exist, and even if it existed it would not be the same file as the one that was listed. We could even create two files named "björn", one named in the UTF-8 encoding and one not. If <c>file:list_dir/1</c> would automatically convert the ISO-latin-1 file name to a list, we would get two identical file names as the result. To avoid this, we need to differentiate between file names being properly encoded according to the Unicode file naming convention (i.e. UTF-8) and file names being invalid under the encoding. This is done by representing invalid encoding as "raw" file names, i.e. as binaries.</p> -<p>The core system of Erlang (Kernel and STDLIB) accepts raw file names except for loadable drivers and executables invoked using <c>open_port({spawn, ...} ...)</c>. <c>open_port({spawn_executable, ...} ...)</c> however does accept them. As mentioned earlier, the arguments given in the option list to <c>open_port({spawn_executable, ...} ...)</c> undergo the same conversion as the file names, meaning that the executable will be provided with arguments in UTF-8 as well. This translation is avoided consistently with how the file names are treated, by giving the argument as a binary.</p> -<p>To force Unicode file name translation mode on systems where this is not the default is considered experimental in OTP R14B01 due to the raw file names possibly being a new experience to the programmer and that the non core applications of OTP are not tested for compliance with raw file names yet. Unicode file name translation is expected to be default in future releases.</p> -<p>If working with raw file names, one can still conform to the encoding convention of the Erlang VM by using the <c>file:native_name_encoding/0</c> function, which returns either the atom <c>latin1</c> or the atom <c>utf8</c> depending on the file name translation mode. On Linux, a VM started without explicitly stating the file name translation mode will default to <c>latin1</c> as the native file name encoding, why file names on the disk encoded as UTF-8 will be returned as a list of the names interpreted as ISO-latin-1. The "UTF-8 list" is not a practical type for displaying or operating on in Erlang, but it is backward compatible and usable in all functions requiring a file name. On Windows and MacOSX, the default behavior is that of file name translation, why the <c>file:native_name_encoding/0</c> by default returns <c>utf8</c> on those systems (the fact that Windows actually does not use UTF-8 on the file system level can safely be ignored by the Erlang programmer). The default behavior can be changed using the <c>+fnu</c> or <c>+fnl</c> options to the VM, see the <seealso marker="erts:erl"><c>erl(1)</c></seealso> command manual page.</p> -<p>Even if you are operating without Unicode file naming translation automatically done by the VM, you can access and create files with names in UTF-8 encoding by using raw file names encoded as UTF-8. Enforcing the UTF-8 encoding regardless of the mode the Erlang VM is started in might, in some circumstances be a good idea, as the convention of using UTF-8 file names is spreading.</p> + <p>Even if you are operating without Unicode file naming translation + automatically done by the VM, you can access and create files with + names in UTF-8 encoding by using raw file names encoded as + UTF-8. Enforcing the UTF-8 encoding regardless of the mode the + Erlang VM is started in might, in some circumstances be a good idea, + as the convention of using UTF-8 file names is spreading.</p> </section> <section> -<title>Notes About MacOSX</title> -<p>MacOSXs vfs layer enforces UTF-8 file names in a quite aggressive way. Older versions did this by simply refusing to create non UTF-8 conforming file names, while newer versions replace offending bytes with the sequence "%HH", where HH is the original character in hexadecimal notation. As Unicode translation is enabled by default on MacOSX, the only way to come up against this is to either start the VM with the <c>+fnl</c> flag or to use a raw file name in <c>latin1</c> encoding. In that case, the file can not be opened with the same name as the one used to create this. The problem is by design in newer versions of MacOSX.</p> -<p>MacOSX also reorganizes the names of files so that the representation of accents etc is denormalized, i.e. the character <c>ö</c> is represented as the codepoints [111,776], where 111 is the character <c>o</c> and 776 is a special accent character. This type of denormalized Unicode is otherwise very seldom used and Erlang normalizes those file names on retrieval, so that denormalized file names is not passed up to the Erlang application. In Erlang the file name "björn" is retrieved as [98,106,246,114,110], not as [98,106,117,776,114,110], even though the file system might think differently.</p> + <title>Notes About MacOS X</title> + <p>MacOS X's vfs layer enforces UTF-8 file names in a quite + aggressive way. Older versions did this by simply refusing to create + non UTF-8 conforming file names, while newer versions replace + offending bytes with the sequence "%HH", where HH is the + original character in hexadecimal notation. As Unicode translation + is enabled by default on MacOS X, the only way to come up against + this is to either start the VM with the <c>+fnl</c> flag or to use a + raw file name in bytewise (<c>latin1</c>) encoding. If using a raw + filename, with a bytewise encoding containing characters between 127 + and 255, to create a file, the file can not be opened using the same + name as the one used to create it. There is no remedy for this + behaviour, other than keeping the file names in the right + encoding.</p> + + <p>MacOS X also reorganizes the names of files so that the + representation of accents etc is using the "combining characters", + i.e. the character <c>ö</c> is represented as the code points + [111,776], where 111 is the character <c>o</c> and 776 is the + special accent character "combining diaeresis". This way of + normalizing Unicode is otherwise very seldom used and Erlang + normalizes those file names in the opposite way upon retrieval, so + that file names using combining accents are not passed up to the + Erlang application. In Erlang the file name "björn" is + retrieved as [98,106,246,114,110], not as [98,106,117,776,114,110], + even though the file system might think differently. The + normalization into combining accents are redone when actually + accessing files, so this can usually be ignored by the Erlang + programmer.</p> </section> </section> <section> -<title>Unicode in Environment Variables and Parameters</title> -<p>Environment variables and their interpretation is handled much in the same way as file names. If Unicode file names are enabled, environment variables as well as parameters to the Erlang VM are expected to be in Unicode.</p> -<p>If Unicode file names are enabled, the calls to <seealso marker="kernel:os#getenv/0"><c>os:getenv/0</c></seealso>, <seealso marker="kernel:os#getenv/1"><c>os:getenv/1</c></seealso> and <seealso marker="kernel:os#putenv/2"><c>os:putenv/2</c></seealso> will handle Unicode strings. On Unix-like platforms, the built-in functions will translate environment variables in UTF-8 to/from Unicode strings, possibly with codepoints > 255. On Windows the Unicode versions of the environment system API will be used, also allowing for codepoints > 255.</p> -<p>On Unix-like operating systems, parameters are expected to be UTF-8 without translation if Unicode file names are enabled.</p> + <title>Unicode in Environment and Parameters</title> + <p>Environment variables and their interpretation is handled much in + the same way as file names. If Unicode file names are enabled, + environment variables as well as parameters to the Erlang VM are + expected to be in Unicode.</p> + <p>If Unicode file names are enabled, the calls to + <seealso marker="kernel:os#getenv/0"><c>os:getenv/0</c></seealso>, + <seealso marker="kernel:os#getenv/1"><c>os:getenv/1</c></seealso> and + <seealso marker="kernel:os#putenv/2"><c>os:putenv/2</c></seealso> + will handle Unicode strings. On Unix-like platforms, the built-in + functions will translate environment variables in UTF-8 to/from + Unicode strings, possibly with code points > 255. On Windows the + Unicode versions of the environment system API will be used, also + allowing for code points > 255.</p> + <p>On Unix-like operating systems, parameters are expected to be + UTF-8 without translation if Unicode file names are enabled.</p> </section> <section> -<title>Unicode-aware Modules</title> -<p>Most of the modules in Erlang/OTP are of course Unicode-unaware in the sense that they have no notion of Unicode and really should not have. Typically they handle non-textual or byte-oriented data (like <c>gen_tcp</c> etc).</p> -<p>Modules that actually handle textual data (like <c>io_lib</c>, <c>string</c> etc) are sometimes subject to conversion or extension to be able to handle Unicode characters.</p> -<p>Fortunately, most textual data has been stored in lists and range checking has been sparse, why modules like <c>string</c> works well for Unicode lists with little need for conversion or extension.</p> -<p>Some modules are however changed to be explicitly Unicode-aware. These modules include:</p> -<taglist> -<tag><c>unicode</c></tag> -<item> -<p>The module <seealso marker="stdlib:unicode">unicode</seealso> is obviously Unicode-aware. It contains functions for conversion between different Unicode formats as well as some utilities for identifying byte order marks. Few programs handling Unicode data will survive without this module.</p> -</item> -<tag><c>io</c></tag> -<item> -<p>The <seealso marker="stdlib:io">io</seealso> module has been extended along with the actual I/O-protocol to handle Unicode data. This means that several functions require binaries to be in UTF-8 and there are modifiers to formatting control sequences to allow for outputting of Unicode strings.</p> -</item> -<tag><c>file</c>, <c>group</c>, <c>user</c></tag> -<item> -<p>I/O-servers throughout the system are able both to handle Unicode data and has options for converting data upon actual output or input to/from the device. As shown earlier, the <seealso marker="stdlib:shell">shell</seealso> has support for Unicode terminals and the <seealso marker="kernel:file">file</seealso> module allows for translation to and from various Unicode formats on disk.</p> -<p>The actual reading and writing of files with Unicode data is however not best done with the <c>file</c> module as its interface is byte oriented. A file opened with a Unicode encoding (like UTF-8), is then best read or written using the <seealso marker="stdlib:io">io</seealso> module.</p> -</item> -<tag><c>re</c></tag> -<item> -<p>The <seealso marker="stdlib:re">re</seealso> module allows for matching Unicode strings as a special option. As the library is actually centered on matching in binaries, the Unicode support is UTF-8-centered.</p> -</item> -<tag><c>wx</c></tag> -<item> -<p>The <seealso marker="wx:wx">wx</seealso> graphical library has extensive support for Unicode text</p> -</item> -</taglist> -<p>The module <seealso marker="stdlib:string">string</seealso> works perfect for Unicode strings as well as for ISO-latin-1 strings with the exception of the language-dependent <seealso marker="stdlib:string#to_upper/1">to_upper</seealso> and <seealso marker="stdlib:string#to_lower/1">to_lower</seealso> functions, which are only correct for the ISO-latin-1 character set. Actually they can never function correctly for Unicode characters in their current form, there are language and locale issues as well as multi-character mappings to consider when conversion text between cases. Converting case in an international environment is a big subject not yet addressed in OTP.</p> + <title>Unicode-aware Modules</title> + <p>Most of the modules in Erlang/OTP are of course Unicode-unaware + in the sense that they have no notion of Unicode and really should + not have. Typically they handle non-textual or byte-oriented data + (like <c>gen_tcp</c> etc).</p> + <p>Modules that actually handle textual data (like <c>io_lib</c>, + <c>string</c> etc) are sometimes subject to conversion or extension + to be able to handle Unicode characters.</p> + <p>Fortunately, most textual data has been stored in lists and range + checking has been sparse, why modules like <c>string</c> works well + for Unicode lists with little need for conversion or extension.</p> + <p>Some modules are however changed to be explicitly + Unicode-aware. These modules include:</p> + <taglist> + <tag><c>unicode</c></tag> + <item> + <p>The module <seealso marker="stdlib:unicode"><c>unicode</c></seealso> + is obviously Unicode-aware. It contains functions for conversion + between different Unicode formats as well as some utilities for + identifying byte order marks. Few programs handling Unicode data + will survive without this module.</p> + </item> + <tag><c>io</c></tag> + <item> + <p>The <seealso marker="stdlib:io"><c>io</c></seealso> module has been + extended along with the actual I/O-protocol to handle Unicode + data. This means that several functions require binaries to be + in UTF-8 and there are modifiers to formatting control sequences + to allow for outputting of Unicode strings.</p> + </item> + <tag><c>file</c>, <c>group</c>, <c>user</c></tag> + <item> + <p>I/O-servers throughout the system are able to handle + Unicode data and has options for converting data upon actual + output or input to/from the device. As shown earlier, the + <seealso marker="stdlib:shell"><c>shell</c></seealso> has support for + Unicode terminals and the <seealso + marker="kernel:file"><c>file</c></seealso> module allows for + translation to and from various Unicode formats on disk.</p> + <p>The actual reading and writing of files with Unicode data is + however not best done with the <c>file</c> module as its + interface is byte oriented. A file opened with a Unicode + encoding (like UTF-8), is then best read or written using the + <seealso marker="stdlib:io"><c>io</c></seealso> module.</p> + </item> + <tag><c>re</c></tag> + <item> + <p>The <seealso marker="stdlib:re"><c>re</c></seealso> module allows + for matching Unicode strings as a special option. As the library + is actually centered on matching in binaries, the Unicode + support is UTF-8-centered.</p> + </item> + <tag><c>wx</c></tag> + <item> + <p>The <seealso marker="wx:wx"><c>wx</c></seealso> graphical library + has extensive support for Unicode text</p> + </item> + </taglist> + <p>The module <seealso + marker="stdlib:string"><c>string</c></seealso> works perfectly for + Unicode strings as well as for ISO-latin-1 strings with the + exception of the language-dependent <seealso + marker="stdlib:string#to_upper/1"><c>to_upper</c></seealso> and + <seealso marker="stdlib:string#to_lower/1"><c>to_lower</c></seealso> + functions, which are only correct for the ISO-latin-1 character + set. Actually they can never function correctly for Unicode + characters in their current form, as there are language and locale + issues as well as multi-character mappings to consider when + converting text between cases. Converting case in an international + environment is a big subject not yet addressed in OTP.</p> </section> <section> -<title>Unicode Recipes</title> -<p>When starting with Unicode, one often stumbles over some common issues. I try to outline some methods of dealing with Unicode data in this section.</p> + <title>Unicode Data in Files</title> + <p>The fact that Erlang as such can handle Unicode data in many forms + does not automatically mean that the content of any file can be + Unicode text. The external entities such as ports or I/O-servers are + not generally Unicode capable.</p> + <p>Ports are always byte oriented, so before sending data that you + are not sure is bytewise encoded to a port, make sure to encode it + in a proper Unicode encoding. Sometimes this will mean that only + part of the data shall be encoded as e.g. UTF-8, some parts may be + binary data (like a length indicator) or something else that shall + not undergo character encoding, so no automatic translation is + present.</p> + <p>I/O-servers behave a little differently. The I/O-servers connected + to terminals (or stdout) can usually cope with Unicode data + regardless of the <c>encoding</c> option. This is convenient when + one expects a modern environment but do not want to crash when + writing to a archaic terminal or pipe. Files on the other hand are + more picky. A file can have an encoding option which makes it + generally usable by the io-module (e.g. <c>{encoding,utf8}</c>), but + is by default opened as a byte oriented file. The <seealso + marker="kernel:file"><c>file</c></seealso> module is byte oriented, why only + ISO-Latin-1 characters can be written using that module. The + <seealso marker="stdlib:io"><c>io</c></seealso> module is the one to use if + Unicode data is to be output to a file with other <c>encoding</c> + than <c>latin1</c> (a.k.a. bytewise encoding). It is slightly + confusing that a file opened with + e.g. <c>file:open(Name,[read,{encoding,utf8}])</c>, cannot be + properly read using <c>file:read(File,N)</c> but you have to use the + <c>io</c> module to retrieve the Unicode data from it. The reason is + that <c>file:read</c> and <c>file:write</c> (and friends) are purely + byte oriented, and should so be, as that is the way to access + files other than text files - byte by byte. Just as with ports, you + can of course write encoded data into a file by "manually" converting + the data to the encoding of choice (using the <seealso + marker="stdlib:unicode"><c>unicode</c></seealso> module or the bit syntax) + and then output it on a bytewise encoded (<c>latin1</c>) file.</p> + <p>The rule of thumb is that the <seealso + marker="kernel:file"><c>file</c></seealso> module should be used for files + opened for bytewise access (<c>{encoding,latin1}</c>) and the + <seealso marker="stdlib:io"><c>io</c></seealso> module should be used when + accessing files with any other encoding + (e.g. <c>{encoding,uf8}</c>).</p> + + <p>Functions reading Erlang syntax from files generally recognize + the <c>coding:</c> comment and can therefore handle Unicode data on + input. When writing Erlang Terms to a file, you should insert + such comments when applicable:</p> + <pre> +$ <input>erl +fna +pc unicode</input> +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] + +Eshell V5.10.1 (abort with ^G) +1> <input>file:write_file("test.term",<<"%% coding: utf-8\n[{\"Юникод\",4711}].\n"/utf8>>).</input> +ok +2> <input>file:consult("test.term").</input> +{ok,[[{"Юникод",4711}]]} + </pre> +</section> +<section> + <title><marker id="unicode_options_summary"/>Summary of Options</title> + <p>The Unicode support is controlled by both command line switches, + some standard environment variables and the version of OTP you are + using. Most options affect mainly the way Unicode data is displayed, + not the actual functionality of the API's in the standard + libraries. This means that Erlang programs usually do not + need to concern themselves with these options, they are more for the + development environment. An Erlang program can be written so that it + works well regardless of the type of system or the Unicode options + that are in effect.</p> + + <p>Here follows a summary of the settings affecting Unicode:</p> + <taglist> + <tag>The <c>LANG</c> and <c>LC_CTYPE</c> environment variables</tag> + <item> + <p>The language setting in the OS mainly affects the shell. The + terminal (i.e. the group leader) will operate with <c>{encoding, + unicode}</c> only if the environment tells it that UTF-8 is + allowed. This setting should correspond to the actual terminal + you are using.</p> + <p>The environment can also affect file name interpretation, if + Erlang is started with the <c>+fna</c> flag.</p> + <p>You can check the setting of this by calling + <c>io:getopts()</c>, which will give you an option list + containing <c>{encoding,unicode}</c> or + <c>{encoding,latin1}</c>.</p> + </item> + <tag>The <c>+pc </c>{<c>unicode</c>|<c>latin1</c>} flag to + <seealso marker="erts:erl"><c>erl(1)</c></seealso></tag> + <item> + <p>This flag affects what is interpreted as string data when + doing heuristic string detection in the shell and in + <c>io</c>/<c>io_lib:format</c> with the <c>"~tp"</c> and + <c>~tP</c> formatting instructions, as described above.</p> + <p>You can check this option by calling io:printable_range/0, + which in R16B will return <c>unicode</c> or <c>latin1</c>. To be + compatible with future (expected) extensions to the settings, + one should rather use <c>io_lib:printable_list/1</c> to check if + a list is printable according to the setting. That function will + take into account new possible settings returned from + <c>io:printable_range/0</c>.</p> + </item> + <tag>The <c>+fn</c>{<c>l</c>|<c>a</c>|<c>u</c>} + [{<c>w</c>|<c>i</c>|<c>e</c>}] + flag to <seealso marker="erts:erl"><c>erl(1)</c></seealso></tag> + <item> + <p>This flag affects how the file names are to be interpreted. On + operating systems with transparent file naming, this has to be + specified to allow for file naming in Unicode characters (and + for correct interpretation of file names containing characters + > 255.</p> + <p><c>+fnl</c> means bytewise interpretation of file names, which + was the usual way to represent ISO-Latin-1 file names before + UTF-8 file naming got widespread. This is the default on all + Unix-like operating systems except MacOS X.</p> + <p><c>+fnu</c> means that file names are encoded in UTF-8, which + is nowadays the common scheme (although not enforced).</p> + <p><c>+fna</c> means that you automatically select between + <c>+fnl</c> and <c>+fnu</c>, based on the <c>LANG</c> and + <c>LC_CTYPE</c> environment variables. This is optimistic + heuristics indeed, nothing enforces a user to have a terminal + with the same encoding as the file system, but usually, this is + the case. This might be the default behavior in a future + release.</p> + + <p>The file name translation mode can be read with the + <c>file:native_name_encoding/0</c> function, which returns + <c>latin1</c> (meaning bytewise encoding) or <c>utf8</c>.</p> + </item> + <tag><seealso marker="stdlib:epp#default_encoding/0"> + <c>epp:default_encoding/0</c></seealso></tag> + <item> + <p>This function returns the default encoding for Erlang source + files (if no encoding comment is present) in the currently + running release. For R16 this returns <c>latin1</c> (meaning + bytewise encoding). In R17 and forward it is expected to return + <c>utf8</c>.</p> + <p>The encoding of each file can be specified using comments as + described in + <seealso marker="stdlib:epp#encoding"><c>epp(3)</c></seealso>.</p> + </item> + <tag><seealso marker="stdlib:io#setopts/1"><c>io:setopts/</c>{<c>1</c>,<c>2</c>}</seealso> and the <c>-oldshell</c>/<c>-noshell</c> flags.</tag> + <item> + <p>When Erlang is started with <c>-oldshell</c> or + <c>-noshell</c>, the I/O-server for <c>standard_io</c> is default + set to bytewise encoding, while an interactive shell defaults to + what the environment variables says.</p> + <p>With the <c>io:setopts/2</c> function you can set the + encoding of a file or other I/O-server. This can also be set when + opening a file. Setting the terminal (or other + <c>standard_io</c> server) unconditionally to the option + <c>{encoding,utf8}</c> will for example make UTF-8 encoded characters + being written to the device regardless of how Erlang was started or + the users environment.</p> + <p>Opening files with <c>encoding</c> option is convenient when + writing or reading text files in a known encoding.</p> + <p>You can retrieve the <c>encoding</c> setting for an I/O-server + using <seealso + marker="stdlib:io#getopts/1"><c>io:getopts()</c></seealso>.</p> + </item> + </taglist> +</section> <section> -<title>Byte Order Marks</title> -<p>A common method of identifying encoding in text-files is to put a byte order mark (BOM) first in the file. The BOM is the codepoint 16#FEFF encoded in the same way as the rest of the file. If such a file is to be read, the first few bytes (depending on encoding) is not part of the actual text. This code outlines how to open a file which is believed to have a BOM and set the files encoding and position for further sequential reading (preferably using the <seealso marker="stdlib:io">io</seealso> module). Note that error handling is omitted from the code:</p> + <title>Recipes</title> + <p>When starting with Unicode, one often stumbles over some common + issues. I try to outline some methods of dealing with Unicode data + in this section.</p> + <section> + <title>Byte Order Marks</title> + <p>A common method of identifying encoding in text-files is to put + a byte order mark (BOM) first in the file. The BOM is the + code point 16#FEFF encoded in the same way as the rest of the + file. If such a file is to be read, the first few bytes (depending + on encoding) is not part of the actual text. This code outlines + how to open a file which is believed to have a BOM and set the + files encoding and position for further sequential reading + (preferably using the <seealso marker="stdlib:io"><c>io</c></seealso> + module). Note that error handling is omitted from the code:</p> <code> open_bom_file_for_reading(File) -> {ok,F} = file:open(File,[read,binary]), @@ -275,8 +1119,15 @@ open_bom_file_for_reading(File) -> io:setopts(F,[{encoding,Type}]), {ok,F}. </code> -<p>The <c>unicode:bom_to_encoding/1</c> function identifies the encoding from a binary of at least four bytes. It returns, along with an term suitable for setting the encoding of the file, the actual length of the BOM, so that the file position can be set accordingly. Note that <c>file:position/2</c> always works on byte-offsets, so that the actual byte-length of the BOM is needed.</p> -<p>To open a file for writing and putting the BOM first is even simpler:</p> + <p>The <c>unicode:bom_to_encoding/1</c> function identifies the + encoding from a binary of at least four bytes. It returns, along + with an term suitable for setting the encoding of the file, the + actual length of the BOM, so that the file position can be set + accordingly. Note that <c>file:position/2</c> always works on + byte-offsets, so that the actual byte-length of the BOM is + needed.</p> + <p>To open a file for writing and putting the BOM first is even + simpler:</p> <code> open_bom_file_for_writing(File,Encoding) -> {ok,F} = file:open(File,[write,binary]), @@ -284,38 +1135,95 @@ open_bom_file_for_writing(File,Encoding) -> io:setopts(F,[{encoding,Encoding}]), {ok,F}. </code> -<p>In both cases the file is then best processed using the <c>io</c> module, as the functions in <c>io</c> can handle codepoints beyond the ISO-latin-1 range.</p> -</section> -<section> -<title>Formatted Input and Output</title> -<p>When reading and writing to Unicode-aware entities, like the User or a file opened for Unicode translation, you will probably want to format text strings using the functions in <seealso marker="stdlib:io">io</seealso> or <seealso marker="stdlib:io_lib">io_lib</seealso>. For backward compatibility reasons, these functions do not accept just any list as a string, but require a special <em>translation modifier</em> when working with Unicode texts. The modifier is <c>t</c>. When applied to the <c>s</c> control character in a formatting string, it accepts all Unicode codepoints and expect binaries to be in UTF-8:</p> -<pre> + <p>In both cases the file is then best processed using the + <c>io</c> module, as the functions in <c>io</c> can handle code + points beyond the ISO-latin-1 range.</p> + </section> + <section> + <title>Formatted I/O</title> + <p>When reading and writing to Unicode-aware entities, like the + User or a file opened for Unicode translation, you will probably + want to format text strings using the functions in <seealso + marker="stdlib:io"><c>io</c></seealso> or <seealso + marker="stdlib:io_lib"><c>io_lib</c></seealso>. For backward + compatibility reasons, these functions do not accept just any list + as a string, but require a special <em>translation modifier</em> + when working with Unicode texts. The modifier is <c>t</c>. When + applied to the <c>s</c> control character in a formatting string, + it accepts all Unicode code points and expect binaries to be in + UTF-8:</p> + <pre> 1> <input>io:format("~ts~n",[<<"åäö"/utf8>>]).</input> åäö ok 2> <input>io:format("~s~n",[<<"åäö"/utf8>>]).</input> åäö ok</pre> -<p>Obviously the second <c>io:format/2</c> gives undesired output because the UTF-8 binary is not in latin1. Because ISO-latin-1 is still the defined character set of Erlang, the non prefixed <c>s</c> control character expects ISO-latin-1 in binaries as well as lists.</p> -<p>As long as the data is always lists, the <c>t</c> modifier can be used for any string, but when binary data is involved, care must be taken to make the right choice of formatting characters.</p> -<p>The function <c>format/2</c> in <c>io_lib</c> behaves similarly. This function is defined to return a deep list of characters and the output could easily be converted to binary data for outputting on a device of any kind by a simple <c>erlang:list_to_binary/1</c>. When the translation modifier is used, the list can however contain characters that cannot be stored in one byte. The call to <c>erlang:list_to_binary/1</c> will in that case fail. However, if the I/O server you want to communicate with is Unicode-aware, the list returned can still be used directly:</p> + <p>Obviously the second <c>io:format/2</c> gives undesired output + because the UTF-8 binary is not in latin1. For backward + compatibility, the non prefixed <c>s</c> control character expects + bytewise encoded ISO-latin-1 characters in binaries and lists + containing only code points < 256.</p> + <p>As long as the data is always lists, the <c>t</c> modifier can + be used for any string, but when binary data is involved, care + must be taken to make the right choice of formatting characters. A + bytewise encoded binary will also be interpreted as a string and + printed even when using <c>~ts</c>, but it might be mistaken for a + valid UTF-8 string and one should therefore avoid using the + <c>~ts</c> control if the binary contains bytewise encoded + characters and not UTF-8.</p> + <p>The function <c>format/2</c> in <c>io_lib</c> behaves + similarly. This function is defined to return a deep list of + characters and the output could easily be converted to binary data + for outputting on a device of any kind by a simple + <c>erlang:list_to_binary/1</c>. When the translation modifier is + used, the list can however contain characters that cannot be + stored in one byte. The call to <c>erlang:list_to_binary/1</c> + will in that case fail. However, if the I/O server you want to + communicate with is Unicode-aware, the list returned can still be + used directly:</p> <pre> -$ <input>erl</input> -Erlang R16B (erts-5.10) [source] [async-threads:0] [hipe] [kernel-poll:false] +$ <input>erl +pc unicode</input> +Erlang R16B (erts-5.10.1) [source] [async-threads:0] [hipe] [kernel-poll:false] -Eshell V5.10 (abort with ^G) +Eshell V5.10.1 (abort with ^G) 1> <input>io_lib:format("~ts~n", ["Γιούνικοντ"]).</input> ["Γιούνικοντ","\n"] 2> <input>io:put_chars(io_lib:format("~ts~n", ["Γιούνικοντ"])).</input> Γιούνικοντ ok</pre> -<p>The Unicode string is returned as a Unicode list, which is recognized as such since the Erlang shell uses the Unicode encoding. The Unicode list is valid input to the <seealso marker="stdlib:io#put_chars/2">io:put_chars/2</seealso> function, so data can be output on any Unicode capable device. If the device is a terminal, characters will be output in the <c>\x{</c>H ...<c>}</c> format if encoding is <c>latin1</c> otherwise in UTF-8 (for the non-interactive terminal - "oldshell" or "noshell") or whatever is suitable to show the character properly (for an interactive terminal - the regular shell). The bottom line is that you can always send Unicode data to the <c>standard_io</c> device. Files will however only accept Unicode codepoints beyond ISO-latin-1 if <c>encoding</c> is set to something else than <c>latin1</c>.</p> -</section> -<section> -<title>Heuristic Identification of UTF-8</title> -<p>While it iss strongly encouraged that the actual encoding of characters in binary data is known prior to processing, that is not always possible. On a typical Linux® system, there is a mix of UTF-8 and ISO-latin-1 text files and there are seldom any BOM's in the files to identify them.</p> -<p>UTF-8 is designed in such a way that ISO-latin-1 characters with numbers beyond the 7-bit ASCII range are seldom considered valid when decoded as UTF-8. Therefore one can usually use heuristics to determine if a file is in UTF-8 or if it is encoded in ISO-latin-1 (one byte per character) encoding. The <c>unicode</c> module can be used to determine if data can be interpreted as UTF-8:</p> -<code> + <p>The Unicode string is returned as a Unicode list, which is + recognized as such since the Erlang shell uses the Unicode + encoding (and is started with all Unicode characters considered + printable). The Unicode list is valid input to the <seealso + marker="stdlib:io#put_chars/2"><c>io:put_chars/2</c></seealso> function, + so data can be output on any Unicode capable device. If the device + is a terminal, characters will be output in the <c>\x{</c>H + ...<c>}</c> format if encoding is <c>latin1</c> otherwise in UTF-8 + (for the non-interactive terminal - "oldshell" or "noshell") or + whatever is suitable to show the character properly (for an + interactive terminal - the regular shell). The bottom line is that + you can always send Unicode data to the <c>standard_io</c> + device. Files will however only accept Unicode code points beyond + ISO-latin-1 if <c>encoding</c> is set to something else than + <c>latin1</c>.</p> + </section> + <section> + <title>Heuristic Identification of UTF-8</title> + <p>While it is + strongly encouraged that the actual encoding of characters in + binary data is known prior to processing, that is not always + possible. On a typical Linux system, there is a mix of UTF-8 + and ISO-latin-1 text files and there are seldom any BOM's in the + files to identify them.</p> + <p>UTF-8 is designed in such a way that ISO-latin-1 characters + with numbers beyond the 7-bit ASCII range are seldom considered + valid when decoded as UTF-8. Therefore one can usually use + heuristics to determine if a file is in UTF-8 or if it is encoded + in ISO-latin-1 (one byte per character) encoding. The + <c>unicode</c> module can be used to determine if data can be + interpreted as UTF-8:</p> + <code> heuristic_encoding_bin(Bin) when is_binary(Bin) -> case unicode:characters_to_binary(Bin,utf8,utf8) of Bin -> @@ -323,9 +1231,16 @@ heuristic_encoding_bin(Bin) when is_binary(Bin) -> _ -> latin1 end. -</code> -<p>If one does not have a complete binary of the file content, one could instead chunk through the file and check part by part. The return-tuple <c>{incomplete,Decoded,Rest}</c> from <c>unicode:characters_to_binary/{1,2,3}</c> comes in handy. The incomplete rest from one chunk of data read from the file is prepended to the next chunk and we therefore circumvent the problem of character boundaries when reading chunks of bytes in UTF-8 encoding:</p> -<code> + </code> + <p>If one does not have a complete binary of the file content, one + could instead chunk through the file and check part by part. The + return-tuple <c>{incomplete,Decoded,Rest}</c> from + <c>unicode:characters_to_binary/{1,2,3}</c> comes in handy. The + incomplete rest from one chunk of data read from the file is + prepended to the next chunk and we therefore circumvent the + problem of character boundaries when reading chunks of bytes in + UTF-8 encoding:</p> + <code> heuristic_encoding_file(FileName) -> {ok,F} = file:open(FileName,[read,binary]), loop_through_file(F,<<>>,file:read(F,1024)). @@ -343,9 +1258,12 @@ loop_through_file(F,Acc,{ok,Bin}) when is_binary(Bin) -> Res when is_binary(Res) -> loop_through_file(F,<<>>,file:read(F,1024)) end. -</code> -<p>Another option is to try to read the whole file in utf8 encoding and see if it fails. Here we need to read the file using <c>io:get_chars/3</c>, as we have to succeed in reading characters with a codepoint over 255:</p> -<code> + </code> + <p>Another option is to try to read the whole file in UTF-8 + encoding and see if it fails. Here we need to read the file using + <c>io:get_chars/3</c>, as we have to succeed in reading characters + with a code point over 255:</p> + <code> heuristic_encoding_file2(FileName) -> {ok,F} = file:open(FileName,[read,binary,{encoding,utf8}]), loop_through_file2(F,io:get_chars(F,'',1024)). @@ -356,7 +1274,68 @@ loop_through_file2(_,{error,_Err}) -> latin1; loop_through_file2(F,Bin) when is_binary(Bin) -> loop_through_file2(F,io:get_chars(F,'',1024)). -</code> -</section> + </code> + </section> + <section> + <title>Lists of UTF-8 Bytes</title> + <p>For various reasons, you may find yourself having a list of + UTF-8 bytes. This is not a regular string of Unicode characters as + each element in the list does not contain one character. Instead + you get the "raw" UTF-8 encoding that you have in binaries. This + is easily converted to a proper Unicode string by first converting + byte per byte into a binary and then converting the binary of + UTF-8 encoded characters back to a Unicode string:</p> + <code> + utf8_list_to_string(StrangeList) -> + unicode:characters_to_list(list_to_binary(StrangeList)). + </code> + </section> + <section> + <title>Double UTF-8 Encoding</title> + <p>When working with binaries, you may get the horrible "double + UTF-8 encoding", where strange characters are encoded in your + binaries or files that you did not expect. What you may have got, + is a UTF-8 encoded binary that is for the second time encoded as + UTF-8. A common situation is where you read a file, byte by byte, + but the actual content is already UTF-8. If you then convert the + bytes to UTF-8, using i.e. the <c>unicode</c> module or by + writing to a file opened with the <c>{encoding,utf8}</c> + option. You will have each <i>byte</i> in the in the input file + encoded as UTF-8, not each character of the original text (one + character may have been encoded in several bytes). There is no + real remedy for this other than being very sure of which data is + actually encoded in which format, and never convert UTF-8 data + (possibly read byte by byte from a file) into UTF-8 again.</p> + <p>The by far most common situation where this happens, is when + you get lists of UTF-8 instead of proper Unicode strings, and then + convert them to UTF-8 in a binary or on a file:</p> + <code> + wrong_thing_to_do() -> + {ok,Bin} = file:read_file("an_utf8_encoded_file.txt"), + MyList = binary_to_list(Bin), %% Wrong! It is an utf8 binary! + {ok,C} = file:open("catastrophe.txt",[write,{encoding,utf8}]), + io:put_chars(C,MyList), %% Expects a Unicode string, but get UTF-8 + %% bytes in a list! + file:close(C). %% The file catastrophe.txt contains more or less unreadable + %% garbage! + </code> + <p>Make very sure you know what a binary contains before + converting it to a string. If no other option exists, try + heuristics:</p> + <code> + if_you_can_not_know() -> + {ok,Bin} = file:read_file("maybe_utf8_encoded_file.txt"), + MyList = case unicode:characters_to_list(Bin) of + L when is_list(L) -> + L; + _ -> + binary_to_list(Bin) %% The file was bytewise encoded + end, + %% Now we know that the list is a Unicode string, not a list of UTF-8 bytes + {ok,G} = file:open("greatness.txt",[write,{encoding,utf8}]), + io:put_chars(G,MyList), %% Expects a Unicode string, which is what it gets! + file:close(G). %% The file contains valid UTF-8 encoded Unicode characters! + </code> + </section> </section> </chapter> diff --git a/lib/stdlib/src/binary.erl b/lib/stdlib/src/binary.erl index 41b6ab1d5f..4850a59eb6 100644 --- a/lib/stdlib/src/binary.erl +++ b/lib/stdlib/src/binary.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2012. All Rights Reserved. +%% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -58,7 +58,7 @@ bin_to_list(_, _) -> -spec bin_to_list(Subject, Pos, Len) -> [byte()] when Subject :: binary(), Pos :: non_neg_integer(), - Len :: non_neg_integer(). + Len :: integer(). bin_to_list(_, _, _) -> erlang:nif_error(undef). @@ -186,7 +186,7 @@ part(_, _) -> -spec part(Subject, Pos, Len) -> binary() when Subject :: binary(), Pos :: non_neg_integer(), - Len :: non_neg_integer(). + Len :: integer(). part(_, _, _) -> erlang:nif_error(undef). diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index e31ae6b9ef..91d317489c 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -512,7 +512,7 @@ m(M) -> print_object_file(Mod) -> case code:is_loaded(Mod) of {file,File} -> - format("Object file: ~s\n", [File]); + format("Object file: ~ts\n", [File]); _ -> ignore end. diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index 06dae51cc9..7c7566e4ec 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -42,7 +42,7 @@ | {encoding, latin1 | unicode | utf8}). -type(options() :: hook_function() | [option()]). --record(pp, {string_fun, char_fun, term_fun}). +-record(pp, {string_fun, char_fun}). -record(options, {hook, encoding, opts}). @@ -182,13 +182,11 @@ state(_Hook) -> state() -> #pp{string_fun = fun io_lib:write_string_as_latin1/1, - char_fun = fun io_lib:write_char_as_latin1/1, - term_fun = fun(T) -> io_lib:format("~p", [T]) end}. + char_fun = fun io_lib:write_char_as_latin1/1}. unicode_state() -> #pp{string_fun = fun io_lib:write_string/1, - char_fun = fun io_lib:write_char/1, - term_fun = fun(T) -> io_lib:format("~tp", [T]) end}. + char_fun = fun io_lib:write_char/1}. encoding(Options) -> case proplists:get_value(encoding, Options, epp:default_encoding()) of @@ -204,10 +202,10 @@ lform({function,Line,Name,Arity,Clauses}, Opts, _State) -> lform({rule,Line,Name,Arity,Clauses}, Opts, _State) -> lrule({rule,Line,Name,Arity,Clauses}, Opts); %% These are specials to make it easier for the compiler. -lform({error,E}, _Opts, State) -> - leaf((State#pp.term_fun)({error,E})++"\n"); -lform({warning,W}, _Opts, State) -> - leaf((State#pp.term_fun)({warning,W})++"\n"); +lform({error,E}, _Opts, _State) -> + leaf(format("~p\n", [{error,E}])); +lform({warning,W}, _Opts, _State) -> + leaf(format("~p\n", [{warning,W}])); lform({eof,_Line}, _Opts, _State) -> $\n. @@ -233,7 +231,7 @@ lattribute(import, Name, _Opts, _State) when is_list(Name) -> lattribute(import, {From,Falist}, _Opts, _State) -> attr("import",[{var,0,pname(From)},falist(Falist)]); lattribute(file, {Name,Line}, _Opts, State) -> - attr("file", [{var,0,(State#pp.term_fun)(Name)},{integer,0,Line}]); + attr("file", [{var,0,(State#pp.string_fun)(Name)},{integer,0,Line}]); lattribute(record, {Name,Is}, Opts, _State) -> Nl = leaf(format("-record(~w,", [Name])), [{first,Nl,record_fields(Is, Opts)},$)]; diff --git a/lib/stdlib/src/filelib.erl b/lib/stdlib/src/filelib.erl index 7df72a93e5..42ef3679a2 100644 --- a/lib/stdlib/src/filelib.erl +++ b/lib/stdlib/src/filelib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl index e944dd4c43..66e54ef221 100644 --- a/lib/stdlib/src/filename.erl +++ b/lib/stdlib/src/filename.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -62,14 +62,14 @@ -spec absname(Filename) -> file:filename_all() when - Filename :: file:name(). + Filename :: file:name_all(). absname(Name) -> {ok, Cwd} = file:get_cwd(), absname(Name, Cwd). -spec absname(Filename, Dir) -> file:filename_all() when - Filename :: file:name(), - Dir :: file:name(). + Filename :: file:name_all(), + Dir :: file:name_all(). absname(Name, AbsBase) when is_binary(Name), is_list(AbsBase) -> absname(Name,filename_string_to_binary(AbsBase)); absname(Name, AbsBase) when is_list(Name), is_binary(AbsBase) -> @@ -123,8 +123,8 @@ absname_vr([[X, $:]|Name], _, _AbsBase) -> %% AbsBase must be absolute and Name must be relative. -spec absname_join(Dir, Filename) -> file:filename_all() when - Dir :: file:name(), - Filename :: file:name(). + Dir :: file:name_all(), + Filename :: file:name_all(). absname_join(AbsBase, Name) -> join(AbsBase, flatten(Name)). @@ -137,7 +137,7 @@ absname_join(AbsBase, Name) -> %% basename("/") -> [] -spec basename(Filename) -> file:filename_all() when - Filename :: file:name(). + Filename :: file:name_all(). basename(Name) when is_binary(Name) -> case os:type() of {win32,_} -> @@ -202,8 +202,8 @@ skip_prefix(Name, _) -> %% rootname(basename("xxx.erl")) -> "xxx" -spec basename(Filename, Ext) -> file:filename_all() when - Filename :: file:name(), - Ext :: file:name(). + Filename :: file:name_all(), + Ext :: file:name_all(). basename(Name, Ext) when is_binary(Name), is_list(Ext) -> basename(Name,filename_string_to_binary(Ext)); basename(Name, Ext) when is_list(Name), is_binary(Ext) -> @@ -252,7 +252,7 @@ basename([], _Ext, Tail, _DrvSep2) -> %% dirname("kalle.erl") -> "." -spec dirname(Filename) -> file:filename_all() when - Filename :: file:name(). + Filename :: file:name_all(). dirname(Name) when is_binary(Name) -> {Dsep,Drivesep} = separators(), SList = case Dsep of @@ -345,7 +345,7 @@ dirjoin1([H|T],Acc,Sep) -> %% On Windows: fn:dirname("\\usr\\src/kalle.erl") -> "/usr/src" -spec extension(Filename) -> file:filename_all() when - Filename :: file:name(). + Filename :: file:name_all(). extension(Name) when is_binary(Name) -> {Dsep,_} = separators(), SList = case Dsep of @@ -388,7 +388,7 @@ extension([], Result, _OsType) -> %% Joins a list of filenames with directory separators. -spec join(Components) -> file:filename_all() when - Components :: [file:name()]. + Components :: [file:name_all()]. join([Name1, Name2|Rest]) -> join([join(Name1, Name2)|Rest]); join([Name]) when is_list(Name) -> @@ -401,8 +401,8 @@ join([Name]) when is_atom(Name) -> %% Joins two filenames with directory separators. -spec join(Name1, Name2) -> file:filename_all() when - Name1 :: file:name(), - Name2 :: file:name(). + Name1 :: file:name_all(), + Name2 :: file:name_all(). join(Name1, Name2) when is_list(Name1), is_list(Name2) -> OsType = major_os_type(), case pathtype(Name2) of @@ -488,7 +488,7 @@ maybe_remove_dirsep(Name, _) -> %% a given base directory, which is is assumed to be normalised %% by a previous call to join/{1,2}. --spec append(file:filename_all(), file:name()) -> file:filename_all(). +-spec append(file:filename_all(), file:name_all()) -> file:filename_all(). append(Dir, Name) when is_binary(Dir), is_binary(Name) -> <<Dir/binary,$/:8,Name/binary>>; append(Dir, Name) when is_binary(Dir) -> @@ -511,7 +511,7 @@ append(Dir, Name) -> %% Example: a:bar.erl, /temp/foo.erl -spec pathtype(Path) -> 'absolute' | 'relative' | 'volumerelative' when - Path :: file:name(). + Path :: file:name_all(). pathtype(Atom) when is_atom(Atom) -> pathtype(atom_to_list(Atom)); pathtype(Name) when is_list(Name) or is_binary(Name) -> @@ -565,7 +565,7 @@ win32_pathtype(_) -> relative. %% rootname("/jam.src/foo.erl") -> "/jam.src/foo" -spec rootname(Filename) -> file:filename_all() when - Filename :: file:name(). + Filename :: file:name_all(). rootname(Name) when is_binary(Name) -> list_to_binary(rootname(binary_to_list(Name))); % No need to handle unicode, . is < 128 rootname(Name0) -> @@ -595,8 +595,8 @@ rootname([], Root, _Ext, _OsType) -> %% rootname("/jam.src/foo.erl", ".erl") -> "/jam.src/foo" -spec rootname(Filename, Ext) -> file:filename_all() when - Filename :: file:name(), - Ext :: file:name(). + Filename :: file:name_all(), + Ext :: file:name_all(). rootname(Name, Ext) when is_binary(Name), is_binary(Ext) -> list_to_binary(rootname(binary_to_list(Name),binary_to_list(Ext))); rootname(Name, Ext) when is_binary(Name) -> @@ -623,8 +623,8 @@ rootname2([Char|Rest], Ext, Result) when is_integer(Char) -> %% split("a:\\msdev\\include") -> ["a:/", "msdev", "include"] -spec split(Filename) -> Components when - Filename :: file:name(), - Components :: [file:name()]. + Filename :: file:name_all(), + Components :: [file:name_all()]. split(Name) when is_binary(Name) -> case os:type() of {win32, _} -> win32_splitb(Name); @@ -718,7 +718,7 @@ split([], Comp, Components, OsType) -> %% name will be normalized as done by join/1. -spec nativename(Path) -> file:filename_all() when - Path :: file:name(). + Path :: file:name_all(). nativename(Name0) -> Name = join([Name0]), %Normalize. case os:type() of @@ -922,7 +922,7 @@ major_os_type() -> %% Flatten a list, also accepting atoms. -spec flatten(Filename) -> file:filename_all() when - Filename :: file:name(). + Filename :: file:name_all(). flatten(Bin) when is_binary(Bin) -> Bin; flatten(List) -> diff --git a/lib/stdlib/src/io.erl b/lib/stdlib/src/io.erl index 3dddb0d6e7..c92e9e3ade 100644 --- a/lib/stdlib/src/io.erl +++ b/lib/stdlib/src/io.erl @@ -32,6 +32,8 @@ parse_erl_exprs/4,parse_erl_form/1,parse_erl_form/2, parse_erl_form/3,parse_erl_form/4]). -export([request/1,request/2,requests/1,requests/2]). +%% Implemented in native code +-export([printable_range/0]). -export_type([device/0, format/0, server_no_data/0]). @@ -66,6 +68,11 @@ o_request(Io, Request, Func) -> Other end. +%% Request what the user considers printable characters +-spec printable_range() -> 'unicode' | 'latin1'. +printable_range() -> + erlang:nif_error(undefined). + %% Put chars takes mixed *unicode* list from R13 onwards. -spec put_chars(CharData) -> 'ok' when CharData :: unicode:chardata(). diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index b7ec848e1e..a9b6d4131e 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -72,7 +72,7 @@ -export([quote_atom/2, char_list/1, latin1_char_list/1, deep_char_list/1, deep_latin1_char_list/1, - printable_list/1, printable_latin1_list/1]). + printable_list/1, printable_latin1_list/1, printable_unicode_list/1]). %% Utilities for collecting characters. -export([collect_chars/3, collect_chars/4, @@ -533,27 +533,45 @@ printable_latin1_list(_) -> false. %Everything else is false %% Return true if CharList is a list of printable characters, else %% false. The notion of printable in Unicode terms is somewhat floating. %% Everything that is not a control character and not invalid unicode -%% will be considered printable. +%% will be considered printable. +%% What the user has noted as printable characters is what actually +%% specifies when this function will return true. If the VM is started +%% with +pc latin1, only the latin1 range will be deemed as printable +%% if on the other hand +pc unicode is given, all characters in the Unicode +%% character set are deemed printable. latin1 is default. -spec printable_list(Term) -> boolean() when Term :: term(). -printable_list([C|Cs]) when is_integer(C), C >= $\040, C =< $\176 -> - printable_list(Cs); -printable_list([C|Cs]) +printable_list(L) -> + %% There will be more alternatives returns from io:printable range + %% in the future. To not have a catch-all clause is deliberate. + case io:printable_range() of + latin1 -> + printable_latin1_list(L); + unicode -> + printable_unicode_list(L) + end. + +-spec printable_unicode_list(Term) -> boolean() when + Term :: term(). + +printable_unicode_list([C|Cs]) when is_integer(C), C >= $\040, C =< $\176 -> + printable_unicode_list(Cs); +printable_unicode_list([C|Cs]) when is_integer(C), C >= 16#A0, C < 16#D800; is_integer(C), C > 16#DFFF, C < 16#FFFE; is_integer(C), C > 16#FFFF, C =< 16#10FFFF -> - printable_list(Cs); -printable_list([$\n|Cs]) -> printable_list(Cs); -printable_list([$\r|Cs]) -> printable_list(Cs); -printable_list([$\t|Cs]) -> printable_list(Cs); -printable_list([$\v|Cs]) -> printable_list(Cs); -printable_list([$\b|Cs]) -> printable_list(Cs); -printable_list([$\f|Cs]) -> printable_list(Cs); -printable_list([$\e|Cs]) -> printable_list(Cs); -printable_list([]) -> true; -printable_list(_) -> false. %Everything else is false + printable_unicode_list(Cs); +printable_unicode_list([$\n|Cs]) -> printable_unicode_list(Cs); +printable_unicode_list([$\r|Cs]) -> printable_unicode_list(Cs); +printable_unicode_list([$\t|Cs]) -> printable_unicode_list(Cs); +printable_unicode_list([$\v|Cs]) -> printable_unicode_list(Cs); +printable_unicode_list([$\b|Cs]) -> printable_unicode_list(Cs); +printable_unicode_list([$\f|Cs]) -> printable_unicode_list(Cs); +printable_unicode_list([$\e|Cs]) -> printable_unicode_list(Cs); +printable_unicode_list([]) -> true; +printable_unicode_list(_) -> false. %Everything else is false %% List = nl() %% Return a list of characters to generate a newline. diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 525b534249..7637ad7a3d 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -485,13 +485,18 @@ printable_bin(Bin, Len, D, latin1) -> false end; printable_bin(Bin, Len, D, _Uni) -> - case printable_unicode(Bin, Len, []) of - {_, <<>>, L} -> - {byte_size(Bin) =:= length(L), L}; - {NC, Bin1, L} when D > 0, Len - NC >= D -> - {byte_size(Bin)-byte_size(Bin1) =:= length(L), true, L}; - {_NC, _Bin, _L} -> - false + case valid_utf8(Bin,Len) of + true -> + case printable_unicode(Bin, Len, [], io:printable_range()) of + {_, <<>>, L} -> + {byte_size(Bin) =:= length(L), L}; + {NC, Bin1, L} when D > 0, Len - NC >= D -> + {byte_size(Bin)-byte_size(Bin1) =:= length(L), true, L}; + {_NC, _Bin, _L} -> + false + end; + false -> + printable_bin(Bin, Len, D, latin1) end. printable_bin1(_Bin, _Start, 0) -> @@ -522,24 +527,36 @@ printable_latin1_list([$\e | Cs], N) -> printable_latin1_list(Cs, N - 1); printable_latin1_list([], _) -> all; printable_latin1_list(_, N) -> N. -printable_unicode(<<C/utf8, R/binary>>=Bin, I, L) when I > 0 -> - case printable_char(C) of +valid_utf8(<<>>,_) -> + true; +valid_utf8(_,0) -> + true; +valid_utf8(<<_/utf8, R/binary>>,N) -> + valid_utf8(R,N-1); +valid_utf8(_,_) -> + false. + +printable_unicode(<<C/utf8, R/binary>>=Bin, I, L, Range) when I > 0 -> + case printable_char(C,Range) of true -> - printable_unicode(R, I - 1, [C | L]); + printable_unicode(R, I - 1, [C | L],Range); false -> {I, Bin, lists:reverse(L)} end; -printable_unicode(Bin, I, L) -> +printable_unicode(Bin, I, L,_) -> {I, Bin, lists:reverse(L)}. -printable_char($\n) -> true; -printable_char($\r) -> true; -printable_char($\t) -> true; -printable_char($\v) -> true; -printable_char($\b) -> true; -printable_char($\f) -> true; -printable_char($\e) -> true; -printable_char(C) -> +printable_char($\n,_) -> true; +printable_char($\r,_) -> true; +printable_char($\t,_) -> true; +printable_char($\v,_) -> true; +printable_char($\b,_) -> true; +printable_char($\f,_) -> true; +printable_char($\e,_) -> true; +printable_char(C,latin1) -> + C >= $\s andalso C =< $~ orelse + C >= 16#A0 andalso C =< 16#FF; +printable_char(C,unicode) -> C >= $\s andalso C =< $~ orelse C >= 16#A0 andalso C < 16#D800 orelse C > 16#DFFF andalso C < 16#FFFE orelse diff --git a/lib/stdlib/src/qlc.erl b/lib/stdlib/src/qlc.erl index 6a27cff589..48f6622565 100644 --- a/lib/stdlib/src/qlc.erl +++ b/lib/stdlib/src/qlc.erl @@ -3709,7 +3709,7 @@ maybe_error_logger(Name, Why) -> Trimmer = fun(M, _F, _A) -> M =:= erl_eval end, Formater = fun(Term, I) -> io_lib:print(Term, I, 80, -1) end, X = lib:format_stacktrace(1, Stacktrace, Trimmer, Formater), - error_logger:Name("qlc: temporary file was needed for ~w\n~s\n", + error_logger:Name("qlc: temporary file was needed for ~w\n~ts\n", [Why, lists:flatten(X)]). expand_stacktrace() -> diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index df66acb97b..c6c706c3a7 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -129,7 +129,7 @@ start_restricted(RShMod) when is_atom(RShMod) -> error_logger:error_report( lists:flatten( io_lib:fwrite( - "Restricted shell module ~w not found: ~"++cs_p() ++"\n", + "Restricted shell module ~w not found: ~tp\n", [RShMod,What]))), Error end. @@ -214,8 +214,7 @@ server(StartSync) -> ok; {RShMod2,What2} -> io:fwrite( - ("Warning! Restricted shell module ~w not found: ~" - ++cs_p()++".\n" + ("Warning! Restricted shell module ~w not found: ~tp.\n" "Only the commands q() and init:stop() will be allowed!\n"), [RShMod2,What2]), application:set_env(stdlib, restricted_shell, ?MODULE) @@ -284,7 +283,12 @@ get_command(Prompt, Eval, Bs, RT, Ds) -> eof; {error,ErrorInfo,_EndPos} -> %% Skip the rest of the line: + Opts = io:getopts(), + TmpOpts = lists:keyreplace(echo, 1, Opts, + {echo, false}), + _ = io:setopts(TmpOpts), _ = io:get_line(''), + _ = io:setopts(Opts), {error,ErrorInfo}; Else -> Else @@ -337,7 +341,7 @@ get_prompt_func() -> end. bad_prompt_func(M) -> - fwrite_severity(benign, "Bad prompt function: ~"++cs_p(), [M]). + fwrite_severity(benign, "Bad prompt function: ~tp", [M]). default_prompt(N) -> %% Don't bother flattening the list irrespective of what the @@ -1380,27 +1384,18 @@ pp(V, I, RT, Enc) -> {record_print_fun, record_print_fun(RT)}] ++ Enc)). -%% Control sequence 'p' possibly with Unicode translation modifier -cs_p() -> - case encoding() of - latin1 -> "p"; - unicode -> "tp" - end. - columns() -> case io:columns() of {ok,N} -> N; _ -> 80 end. - encoding() -> [{encoding, Encoding}] = enc(), Encoding. - enc() -> case lists:keyfind(encoding, 1, io:getopts()) of - false -> [{encoding,latin1}]; % should never happen - Enc -> [Enc] + false -> [{encoding,latin1}]; % should never happen + Enc -> [Enc] end. garb(Shell) -> @@ -1424,10 +1419,9 @@ check_env(V) -> {ok, Val} when is_integer(Val), Val >= 0 -> ok; {ok, Val} -> - Txt = io_lib:fwrite( - ("Invalid value of STDLIB configuration parameter ~w: ~" - ++cs_p()++"\n"), - [V, Val]), + Txt = io_lib:fwrite + ("Invalid value of STDLIB configuration parameter" + "~w: ~tp\n", [V, Val]), error_logger:info_report(lists:flatten(Txt)) end. diff --git a/lib/stdlib/test/epp_SUITE.erl b/lib/stdlib/test/epp_SUITE.erl index 041d521514..b2f1aa955a 100644 --- a/lib/stdlib/test/epp_SUITE.erl +++ b/lib/stdlib/test/epp_SUITE.erl @@ -25,7 +25,7 @@ variable_1/1, otp_4870/1, otp_4871/1, otp_5362/1, pmod/1, not_circular/1, skip_header/1, otp_6277/1, otp_7702/1, otp_8130/1, overload_mac/1, otp_8388/1, otp_8470/1, otp_8503/1, - otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1]). + otp_8562/1, otp_8665/1, otp_8911/1, otp_10302/1, otp_10820/1]). -export([epp_parse_erl_form/2]). @@ -67,7 +67,7 @@ all() -> {group, variable}, otp_4870, otp_4871, otp_5362, pmod, not_circular, skip_header, otp_6277, otp_7702, otp_8130, overload_mac, otp_8388, otp_8470, otp_8503, otp_8562, - otp_8665, otp_8911, otp_10302]. + otp_8665, otp_8911, otp_10302, otp_10820]. groups() -> [{upcase_mac, [], [upcase_mac_1, upcase_mac_2]}, @@ -1359,6 +1359,30 @@ encoding_nocom(Enc, File) -> ok = file:close(Fd), E = epp:read_encoding(File, Options). +otp_10820(doc) -> + "OTP-10820. Unicode filenames."; +otp_10820(suite) -> + []; +otp_10820(Config) when is_list(Config) -> + L = [915,953,959,973,957,953,954,959,957,964], + Dir = ?config(priv_dir, Config), + File = filename:join(Dir, L++".erl"), + C1 = <<"%% coding: utf-8\n -module(any).">>, + ok = do_otp_10820(File, C1, "+pc latin1"), + ok = do_otp_10820(File, C1, "+pc unicode"), + C2 = <<"\n-module(any).">>, + ok = do_otp_10820(File, C2, "+pc latin1"), + ok = do_otp_10820(File, C2, "+pc unicode"). + +do_otp_10820(File, C, PC) -> + {ok,Node} = start_node(erl_pp_helper, "+fnu " ++ PC), + ok = rpc:call(Node, file, write_file, [File, C]), + {ok,[{attribute,1,file,{File,1}}, + {attribute,2,module,any}, + {eof,2}]} = rpc:call(Node, epp, parse_file, [File, [],[]]), + true = test_server:stop_node(Node), + ok. + check(Config, Tests) -> eval_tests(Config, fun check_test/2, Tests). @@ -1475,3 +1499,8 @@ ln2({error,M}) -> {error,ln2(M)}; ln2(M) -> M. + +%% +fnu means a peer node has to be started; slave will not do +start_node(Name, Xargs) -> + ?line PA = filename:dirname(code:which(?MODULE)), + test_server:start_node(Name, peer, [{args, "-pa " ++ PA ++ " " ++ Xargs}]). diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index 36229b6989..f8345559c4 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% Copyright Ericsson AB 1999-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 37be61d665..9c0a43abcc 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2012. All Rights Reserved. +%% Copyright Ericsson AB 2006-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,7 +49,7 @@ otp_6321/1, otp_6911/1, otp_6914/1, otp_8150/1, otp_8238/1, otp_8473/1, otp_8522/1, otp_8567/1, otp_8664/1, otp_9147/1, - otp_10302/1]). + otp_10302/1, otp_10820/1]). %% Internal export. -export([ehook/6]). @@ -80,7 +80,8 @@ groups() -> {attributes, [], [misc_attrs, import_export]}, {tickets, [], [otp_6321, otp_6911, otp_6914, otp_8150, otp_8238, - otp_8473, otp_8522, otp_8567, otp_8664, otp_9147, otp_10302]}]. + otp_8473, otp_8522, otp_8567, otp_8664, otp_9147, + otp_10302, otp_10820]}]. init_per_suite(Config) -> Config. @@ -1074,6 +1075,34 @@ otp_10302(Config) when is_list(Config) -> unicode_hook({foo,E}, I, P, H) -> erl_pp:expr({call,0,{atom,0,foo},[E]}, I, P, H). +otp_10820(doc) -> + "OTP-10820. Unicode filenames."; +otp_10820(suite) -> []; +otp_10820(Config) when is_list(Config) -> + C1 = <<"%% coding: utf-8\n -module(any).">>, + ok = do_otp_10820(Config, C1, "+pc latin1"), + ok = do_otp_10820(Config, C1, "+pc unicode"), + C2 = <<"-module(any).">>, + ok = do_otp_10820(Config, C2, "+pc latin1"), + ok = do_otp_10820(Config, C2, "+pc unicode"). + +do_otp_10820(Config, C, PC) -> + {ok,Node} = start_node(erl_pp_helper, "+fnu " ++ PC), + L = [915,953,959,973,957,953,954,959,957,964], + FileName = filename(L++".erl", Config), + ok = rpc:call(Node, file, write_file, [FileName, C]), + {ok, _, []} = rpc:call(Node, compile, file, + [FileName, [return,'P',{outdir,?privdir}]]), + PFileName = filename(L++".P", Config), + {ok, Bin} = rpc:call(Node, file, read_file, [PFileName]), + true = test_server:stop_node(Node), + true = file_attr_is_string(binary_to_list(Bin)), + ok. + +file_attr_is_string("-file(\"" ++ _) -> true; +file_attr_is_string([_ | L]) -> + file_attr_is_string(L). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% compile(Config, Tests) -> @@ -1247,3 +1276,8 @@ filename(Name, Config) -> fail() -> io:format("failed~n"), ?t:fail(). + +%% +fnu means a peer node has to be started; slave will not do +start_node(Name, Xargs) -> + ?line PA = filename:dirname(code:which(?MODULE)), + test_server:start_node(Name, peer, [{args, "-pa " ++ PA ++ " " ++ Xargs}]). diff --git a/lib/stdlib/test/escript_SUITE.erl b/lib/stdlib/test/escript_SUITE.erl index be8fb1b37a..cf5fb12686 100644 --- a/lib/stdlib/test/escript_SUITE.erl +++ b/lib/stdlib/test/escript_SUITE.erl @@ -913,7 +913,7 @@ unicode(Config) when is_list(Config) -> run(Dir, "unicode1", [<<"escript: exception error: an error occurred when evaluating" " an arithmetic expression\n in operator '/'/2\n " - "called as <<170>> / <<170>>\nExitCode:127">>]), + "called as <<224,170,170>> / <<224,170,170>>\nExitCode:127">>]), run(Dir, "unicode2", [<<"escript: exception error: an error occurred when evaluating" " an arithmetic expression\n in operator '/'/2\n " diff --git a/lib/stdlib/test/escript_SUITE_data/unicode1 b/lib/stdlib/test/escript_SUITE_data/unicode1 index a77574625e..351bb785e5 100755 --- a/lib/stdlib/test/escript_SUITE_data/unicode1 +++ b/lib/stdlib/test/escript_SUITE_data/unicode1 @@ -6,7 +6,7 @@ main(_) -> ok = io:setopts([{encoding,unicode}]), _D = erlang:system_flag(backtrace_depth, 0), - A = <<"\x{aa}">>, + A = <<"\x{aaa}"/utf8>>, S = lists:flatten(io_lib:format("~p/~p.", [A, A])), {ok, Ts, _} = erl_scan:string(S, 1, [unicode]), {ok, Es} = erl_parse:parse_exprs(Ts), diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index dc17e5d33c..b6c94a6f6d 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -96,7 +96,7 @@ misc1_do/1, safe_fixtable_do/1, info_do/1, dups_do/1, heavy_lookup_do/1, heavy_lookup_element_do/1, member_do/1, otp_5340_do/1, otp_7665_do/1, meta_wb_do/1, do_heavy_concurrent/1, tab2file2_do/2, exit_large_table_owner_do/2, - types_do/1, sleeper/0, rpc_externals/0, memory_do/1, + types_do/1, sleeper/0, memory_do/1, ms_tracee_dummy/1, ms_tracee_dummy/2, ms_tracee_dummy/3, ms_tracee_dummy/4 ]). @@ -5989,33 +5989,103 @@ make_ext_ref() -> init_externals() -> case get(externals) of undefined -> - SysDistSz = ets:info(sys_dist,size), - ?line Pa = filename:dirname(code:which(?MODULE)), - ?line {ok, Node} = test_server:start_node(plopp, slave, [{args, " -pa " ++ Pa}]), - ?line Res = case rpc:call(Node, ?MODULE, rpc_externals, []) of - {badrpc, {'EXIT', E}} -> - test_server:fail({rpcresult, E}); - R -> R - end, - ?line test_server:stop_node(Node), - - %% Wait for table 'sys_dist' to stabilize - repeat_while(fun() -> - case ets:info(sys_dist,size) of - SysDistSz -> false; - Sz -> - io:format("Waiting for sys_dist to revert size from ~p to size ~p\n", - [Sz, SysDistSz]), - receive after 1000 -> true end - end - end), + OtherNode = {gurka@sallad, 1}, + Res = {mk_pid(OtherNode, 7645, 8123), + mk_port(OtherNode, 187489773), + mk_ref(OtherNode, [262143, 1293964255, 3291964278])}, put(externals, Res); {_,_,_} -> ok end. -rpc_externals() -> - {self(), make_port(), make_ref()}. +%% +%% Node container constructor functions +%% + +-define(VERSION_MAGIC, 131). +-define(PORT_EXT, 102). +-define(PID_EXT, 103). +-define(NEW_REFERENCE_EXT, 114). + +uint32_be(Uint) when is_integer(Uint), 0 =< Uint, Uint < 1 bsl 32 -> + [(Uint bsr 24) band 16#ff, + (Uint bsr 16) band 16#ff, + (Uint bsr 8) band 16#ff, + Uint band 16#ff]; +uint32_be(Uint) -> + exit({badarg, uint32_be, [Uint]}). + +uint16_be(Uint) when is_integer(Uint), 0 =< Uint, Uint < 1 bsl 16 -> + [(Uint bsr 8) band 16#ff, + Uint band 16#ff]; +uint16_be(Uint) -> + exit({badarg, uint16_be, [Uint]}). + +uint8(Uint) when is_integer(Uint), 0 =< Uint, Uint < 1 bsl 8 -> + Uint band 16#ff; +uint8(Uint) -> + exit({badarg, uint8, [Uint]}). + +mk_pid({NodeName, Creation}, Number, Serial) when is_atom(NodeName) -> + <<?VERSION_MAGIC, NodeNameExt/binary>> = term_to_binary(NodeName), + mk_pid({NodeNameExt, Creation}, Number, Serial); +mk_pid({NodeNameExt, Creation}, Number, Serial) -> + case catch binary_to_term(list_to_binary([?VERSION_MAGIC, + ?PID_EXT, + NodeNameExt, + uint32_be(Number), + uint32_be(Serial), + uint8(Creation)])) of + Pid when is_pid(Pid) -> + Pid; + {'EXIT', {badarg, _}} -> + exit({badarg, mk_pid, [{NodeNameExt, Creation}, Number, Serial]}); + Other -> + exit({unexpected_binary_to_term_result, Other}) + end. + +mk_port({NodeName, Creation}, Number) when is_atom(NodeName) -> + <<?VERSION_MAGIC, NodeNameExt/binary>> = term_to_binary(NodeName), + mk_port({NodeNameExt, Creation}, Number); +mk_port({NodeNameExt, Creation}, Number) -> + case catch binary_to_term(list_to_binary([?VERSION_MAGIC, + ?PORT_EXT, + NodeNameExt, + uint32_be(Number), + uint8(Creation)])) of + Port when is_port(Port) -> + Port; + {'EXIT', {badarg, _}} -> + exit({badarg, mk_port, [{NodeNameExt, Creation}, Number]}); + Other -> + exit({unexpected_binary_to_term_result, Other}) + end. + +mk_ref({NodeName, Creation}, Numbers) when is_atom(NodeName), + is_integer(Creation), + is_list(Numbers) -> + <<?VERSION_MAGIC, NodeNameExt/binary>> = term_to_binary(NodeName), + mk_ref({NodeNameExt, Creation}, Numbers); +mk_ref({NodeNameExt, Creation}, Numbers) when is_binary(NodeNameExt), + is_integer(Creation), + is_list(Numbers) -> + case catch binary_to_term(list_to_binary([?VERSION_MAGIC, + ?NEW_REFERENCE_EXT, + uint16_be(length(Numbers)), + NodeNameExt, + uint8(Creation), + lists:map(fun (N) -> + uint32_be(N) + end, + Numbers)])) of + Ref when is_reference(Ref) -> + Ref; + {'EXIT', {badarg, _}} -> + exit({badarg, mk_ref, [{NodeNameExt, Creation}, Numbers]}); + Other -> + exit({unexpected_binary_to_term_result, Other}) + end. + make_sub_binary(Bin) when is_binary(Bin) -> {_,B} = split_binary(list_to_binary([0,1,3,Bin]), 3), diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index aa698ecaa2..9f828c6d2d 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -29,9 +29,12 @@ manpage/1, otp_6708/1, otp_7084/1, otp_7421/1, io_lib_collect_line_3_wb/1, cr_whitespace_in_string/1, io_fread_newlines/1, otp_8989/1, io_lib_fread_literal/1, + printable_range/1, io_lib_print_binary_depth_one/1, otp_10302/1, otp_10755/1, otp_10836/1]). +-export([pretty/2]). + %-define(debug, true). -ifdef(debug). @@ -66,6 +69,7 @@ all() -> manpage, otp_6708, otp_7084, otp_7421, io_lib_collect_line_3_wb, cr_whitespace_in_string, io_fread_newlines, otp_8989, io_lib_fread_literal, + printable_range, io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836]. groups() -> @@ -2026,6 +2030,79 @@ io_lib_fread_literal(Suite) when is_list(Suite) -> ?line {done,{ok,[]},[]} = io_lib:fread(C2, "d\n", " d"), ok. + +printable_range(doc) -> + "Check that the printable range set by the user actually works"; +printable_range(Suite) when is_list(Suite) -> + Pa = filename:dirname(code:which(?MODULE)), + {ok, UNode} = test_server:start_node(printable_range_unicode, slave, + [{args, " +pc unicode -pa " ++ Pa}]), + {ok, LNode} = test_server:start_node(printable_range_latin1, slave, + [{args, " +pc latin1 -pa " ++ Pa}]), + {ok, DNode} = test_server:start_node(printable_range_default, slave, + [{args, " -pa " ++ Pa}]), + unicode = rpc:call(UNode,io,printable_range,[]), + latin1 = rpc:call(LNode,io,printable_range,[]), + latin1 = rpc:call(DNode,io,printable_range,[]), + test_server:stop_node(UNode), + test_server:stop_node(LNode), + {ok, UNode} = test_server:start_node(printable_range_unicode, slave, + [{args, " +pcunicode -pa " ++ Pa}]), + {ok, LNode} = test_server:start_node(printable_range_latin1, slave, + [{args, " +pclatin1 -pa " ++ Pa}]), + unicode = rpc:call(UNode,io,printable_range,[]), + latin1 = rpc:call(LNode,io,printable_range,[]), + {error, _} = test_server:start_node(printable_range_unnicode, slave, + [{args, " +pcunnicode -pa " ++ Pa}]), + PrettyOptions = [{column,1}, + {line_length,109}, + {depth,30}, + {max_chars,60}, + {record_print_fun, + fun(_,_) -> no end}, + {encoding,unicode}], + 1025 = lists:max(lists:flatten(rpc:call(UNode,io_lib_pretty,print, + [{hello, [1024,1025]}, + PrettyOptions]))), + 125 = lists:max(lists:flatten(rpc:call(LNode,io_lib_pretty,print, + [{hello, [1024,1025]}, + PrettyOptions]))), + 125 = lists:max(lists:flatten(rpc:call(DNode,io_lib_pretty,print, + [{hello, [1024,1025]}, + PrettyOptions]))), + 1025 = lists:max(lists:flatten(rpc:call(UNode,io_lib_pretty,print, + [{hello, <<1024/utf8,1025/utf8>>}, + PrettyOptions]))), + 125 = lists:max(lists:flatten(rpc:call(LNode,io_lib_pretty,print, + [{hello, <<1024/utf8,1025/utf8>>}, + PrettyOptions]))), + 125 = lists:max(lists:flatten(rpc:call(DNode,io_lib_pretty,print, + [{hello, <<1024/utf8,1025/utf8>>}, + PrettyOptions]))), + + 1025 = lists:max(lists:flatten(rpc:call(UNode,io_lib,format, + ["~tp",[{hello, [1024,1025]}]]))), + 125 = lists:max(lists:flatten(rpc:call(LNode,io_lib,format, + ["~tp",[{hello, [1024,1025]}]]))), + 125 = lists:max(lists:flatten(rpc:call(DNode,io_lib,format, + ["~tp",[{hello, [1024,1025]}]]))), + 1025 = lists:max(lists:flatten(rpc:call(UNode,io_lib,format, + ["~tp", + [{hello, + <<1024/utf8,1025/utf8>>}]]))), + 125 = lists:max(lists:flatten(rpc:call(LNode,io_lib,format, + ["~tp", + [{hello, + <<1024/utf8,1025/utf8>>}]]))), + 125 = lists:max(lists:flatten(rpc:call(DNode,io_lib,format, + ["~tp", + [{hello, + <<1024/utf8,1025/utf8>>}]]))), + test_server:stop_node(UNode), + test_server:stop_node(LNode), + test_server:stop_node(DNode), + ok. + io_lib_print_binary_depth_one(doc) -> "Test binaries printed with a depth of one behave correctly"; io_lib_print_binary_depth_one(Suite) when is_list(Suite) -> @@ -2040,16 +2117,30 @@ io_lib_print_binary_depth_one(Suite) when is_list(Suite) -> otp_10302(doc) -> "OTP-10302. Unicode"; otp_10302(Suite) when is_list(Suite) -> - "\"\x{400}\"" = pretty("\x{400}", -1), - "<<\"\x{400}\"/utf8>>" = pretty(<<"\x{400}"/utf8>>, -1), + Pa = filename:dirname(code:which(?MODULE)), + {ok, UNode} = test_server:start_node(printable_range_unicode, slave, + [{args, " +pc unicode -pa " ++ Pa}]), + {ok, LNode} = test_server:start_node(printable_range_latin1, slave, + [{args, " +pc latin1 -pa " ++ Pa}]), + "\"\x{400}\"" = rpc:call(UNode,?MODULE,pretty,["\x{400}", -1]), + "<<\"\x{400}\"/utf8>>" = rpc:call(UNode,?MODULE,pretty, + [<<"\x{400}"/utf8>>, -1]), + + "<<\"\x{400}foo\"/utf8>>" = rpc:call(UNode,?MODULE,pretty, + [<<"\x{400}foo"/utf8>>, 2]), + "[1024]" = rpc:call(LNode,?MODULE,pretty,["\x{400}", -1]), + "<<208,128>>" = rpc:call(LNode,?MODULE,pretty,[<<"\x{400}"/utf8>>, -1]), + + "<<208,...>>" = rpc:call(LNode,?MODULE,pretty,[<<"\x{400}foo"/utf8>>, 2]), + test_server:stop_node(UNode), + test_server:stop_node(LNode), - "<<\"\x{400}foo\"/utf8>>" = pretty(<<"\x{400}foo"/utf8>>, 2), "<<\"äppl\"/utf8>>" = pretty(<<"äppl"/utf8>>, 2), "<<\"äppl\"/utf8...>>" = pretty(<<"äpple"/utf8>>, 2), "<<\"apel\">>" = pretty(<<"apel">>, 2), "<<\"apel\"...>>" = pretty(<<"apelsin">>, 2), - "<<228,112,112,108>>" = fmt("~tp", [<<"äppl">>]), - "<<228,...>>" = fmt("~tP", [<<"äppl">>, 2]), + "<<\"äppl\">>" = fmt("~tp", [<<"äppl">>]), + "<<\"äppl\"...>>" = fmt("~tP", [<<"äpple">>, 2]), "<<0,0,0,0,0,0,1,0>>" = fmt("~p", [<<256:64/unsigned-integer>>]), "<<0,0,0,0,0,0,1,0>>" = fmt("~tp", [<<256:64/unsigned-integer>>]), diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index 299daf0e42..4ca8680fc9 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -716,7 +716,7 @@ binary_options(Config) when is_list(Config) -> {getline, "<<\"hej\\n\">>"}, {putline, "io:get_line('')."}, {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, - {getline, "<<\""++binary_to_list(unicode:characters_to_binary(<<"\345\344\366"/utf8>>,latin1,utf8))++"\\n\">>"} + {getline, "<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\n\"/utf8>>"} ],[]) end, %% And one with oldshell @@ -1784,8 +1784,8 @@ get_default_shell() -> {putline, "whereis(user_drv)."}, {getline, "undefined"}],[]), old - catch E:R -> - ?dbg({E,R}), + catch _E:_R -> + ?dbg({_E,_R}), new end. diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index f22df96697..990b1f5eb2 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2012. All Rights Reserved. +%% Copyright Ericsson AB 2004-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -2742,6 +2742,9 @@ otp_10302(doc) -> "OTP-10302. Unicode."; otp_10302(suite) -> []; otp_10302(Config) when is_list(Config) -> + {ok,Node} = start_node(shell_suite_helper_2, + "-pa "++?config(priv_dir,Config)++ + " +pc unicode"), Test1 = <<"begin io:setopts([{encoding,utf8}]), @@ -2749,13 +2752,13 @@ otp_10302(Config) when is_list(Config) -> rd(rec, {a = \"\\x{400}\"}), ok = rl(rec) end.">>, - "-record(rec,{a = \"\x{400}\"}).\nok.\n" = t(Test1), + "-record(rec,{a = \"\x{400}\"}).\nok.\n" = t({Node,Test1}), Test3 = <<"io:setopts([{encoding,utf8}]). rd(rec, {a = \"\\x{400}\"}). ok = rp(#rec{}).">>, - "ok.\nrec\n#rec{a = \"\x{400}\"}.\nok.\n" = t(Test3), + "ok.\nrec\n#rec{a = \"\x{400}\"}.\nok.\n" = t({Node,Test3}), Test4 = <<"io:setopts([{encoding,utf8}]). @@ -2766,7 +2769,7 @@ otp_10302(Config) when is_list(Config) -> "ok.\n\"\x{400}\"\nA = \"\x{400}\".\nok.\n" "1: io:setopts([{encoding,utf8}])\n-> ok.\n" "2: A = [1024] = \"\x{400}\"\n-> \"\x{400}\"\n" - "3: b()\n-> ok.\nok.\n" = t(Test4), + "3: b()\n-> ok.\nok.\n" = t({Node,Test4}), Test5 = <<"begin @@ -2776,18 +2779,20 @@ otp_10302(Config) when is_list(Config) -> b(), h() end.">>, - "A = \"\x{400}\".\nok.\n" = t(Test5), + "A = \"\x{400}\".\nok.\n" = t({Node,Test5}), %% One $" is "lost": true = "\x{400}\": command not found" =:= - prompt_err({<<"io:setopts([{encoding,utf8}]). v(\"\x{400}\")."/utf8>>, + prompt_err({Node, + <<"io:setopts([{encoding,utf8}]). v(\"\x{400}\")."/utf8>>, unicode}), "ok.\ndefault\n* Bad prompt function: \"\x{400}\".\n" = - t({<<"io:setopts([{encoding,utf8}]). " + t({Node,<<"io:setopts([{encoding,utf8}]). " "shell:prompt_func(\"\x{400}\")."/utf8>>, unicode}), + rpc:call(Node,shell, prompt_func, [default]), _ = shell:prompt_func(default), %% Test lib:format_exception() (cf. OTP-6554) @@ -2812,10 +2817,10 @@ otp_10302(Config) when is_list(Config) -> {ok, Es} = erl_parse:parse_exprs(Ts), B = erl_eval:new_bindings(), erl_eval:exprs(Es, B).">>, - + "ok.\n** exception error: an error occurred when evaluating" " an arithmetic expression\n in operator '/'/2\n" - " called as <<170>> / <<170>>.\n" = t(Test7), + " called as <<\"�\">> / <<\"�\">>.\n" = t({Node,Test7}), Test8 = <<"begin A = [1089], @@ -2839,7 +2844,7 @@ otp_10302(Config) when is_list(Config) -> "ok.\n** exception error: an error occurred when evaluating" " an arithmetic expression\n in operator '/'/2\n" - " called as \"\x{441}\" / \"\x{441}\".\n" = t(Test9), + " called as \"\x{441}\" / \"\x{441}\".\n" = t({Node,Test9}), Test10 = <<"A = {\"1\\xaa\", $\\xaa, @@ -2861,7 +2866,7 @@ otp_10302(Config) when is_list(Config) -> "ok.\n** exception error: no function clause matching \n" " erl_eval:'-inside-an-interpreted-fun-'" "({\"1\xaa\",170,<<\"hi\">>,\n " - " <<\"1\xaa\"/utf8>>}) .\n" = t(Test11), + " <<\"1\xaa\"/utf8>>}) .\n" = t({Node,Test11}), Test12 = <<"fun(a, b) -> false end(65, [1089]).">>, "** exception error: no function clause matching \n" " erl_eval:'-inside-an-interpreted-fun-'(65,[1089])" @@ -2871,8 +2876,9 @@ otp_10302(Config) when is_list(Config) -> fun(a, b) -> false end(65, [1089]).">>, "ok.\n** exception error: no function clause matching \n" " erl_eval:'-inside-an-interpreted-fun-'(65,\"\x{441}\")" - " .\n" = t(Test13), + " .\n" = t({Node,Test13}), + test_server:stop_node(Node), ok. scan(B) -> @@ -2895,6 +2901,8 @@ scan(S0, F) -> [] end. +t({Node,Bin,Enc}) when is_atom(Node),is_binary(Bin), is_atom(Enc) -> + t0({Bin,Enc}, fun() -> start_new_shell(Node) end); t({Node,Bin}) when is_atom(Node),is_binary(Bin) -> t0({Bin,latin1}, fun() -> start_new_shell(Node) end); t(Bin) when is_binary(Bin) -> diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index c1467697e3..fbb838c686 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 1.19.1 +STDLIB_VSN = 1.19.2 diff --git a/lib/syntax_tools/doc/src/notes.xml b/lib/syntax_tools/doc/src/notes.xml index f5afcfa413..8ae69539a7 100644 --- a/lib/syntax_tools/doc/src/notes.xml +++ b/lib/syntax_tools/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2007</year><year>2012</year> + <year>2007</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,6 +31,20 @@ <p>This document describes the changes made to the Syntax_Tools application.</p> +<section><title>Syntax_Tools 1.6.11</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Miscellaneous updates due to Unicode support. </p> + <p> + Own Id: OTP-10820</p> + </item> + </list> + </section> + +</section> + <section><title>Syntax_Tools 1.6.10</title> <section><title>Improvements and New Features</title> diff --git a/lib/syntax_tools/src/erl_comment_scan.erl b/lib/syntax_tools/src/erl_comment_scan.erl index b949675f1c..dae7530ce7 100644 --- a/lib/syntax_tools/src/erl_comment_scan.erl +++ b/lib/syntax_tools/src/erl_comment_scan.erl @@ -291,7 +291,7 @@ filename(N) -> exit(error). error_read_file(Name) -> - report_error("error reading file `~s'.", [Name]). + report_error("error reading file `~ts'.", [Name]). report_error(S, Vs) -> error_logger:error_msg(lists:concat([?MODULE, ": ", S, "\n"]), Vs). diff --git a/lib/syntax_tools/src/erl_tidy.erl b/lib/syntax_tools/src/erl_tidy.erl index 35fd0c2e02..0c149634f6 100644 --- a/lib/syntax_tools/src/erl_tidy.erl +++ b/lib/syntax_tools/src/erl_tidy.erl @@ -151,7 +151,7 @@ dir_1(Dir, Regexp, Env) -> lists:foreach(fun (X) -> dir_2(X, Regexp, Dir, Env) end, Files); {error, _} -> - report_error("error reading directory `~s'", + report_error("error reading directory `~ts'", [filename(Dir)]), exit(error) end. @@ -180,7 +180,7 @@ dir_2(Name, Regexp, Dir, Env) -> dir_3(Name, Dir, Regexp, Env) -> Dir1 = filename:join(Dir, Name), - verbose("tidying directory `~s'.", [Dir1], Env#dir.options), + verbose("tidying directory `~ts'.", [Dir1], Env#dir.options), dir_1(Dir1, Regexp, Env). dir_4(File, Regexp, Env) -> @@ -189,7 +189,7 @@ dir_4(File, Regexp, Env) -> Opts = [{outfile, File}, {dir, ""} | Env#dir.options], case catch file(File, Opts) of {'EXIT', Value} -> - warn("error tidying `~s'.~n~p", [File,Value], Opts); + warn("error tidying `~ts'.~n~p", [File,Value], Opts); _ -> ok end; @@ -314,7 +314,7 @@ file_2(Name, Opts) -> end. read_module(Name, Opts) -> - verbose("reading module `~s'.", [filename(Name)], Opts), + verbose("reading module `~ts'.", [filename(Name)], Opts), case epp_dodger:parse_file(Name, [no_fail]) of {ok, Forms} -> check_forms(Forms, Name), @@ -335,7 +335,7 @@ check_forms(Fs, Name) -> "unknown error" end, report_error({Name, erl_syntax:get_pos(F), - "\n ~s"}, [S]), + "\n ~ts"}, [S]), exit(error); _ -> ok @@ -357,18 +357,18 @@ write_module(Tree, Name, Opts) -> {value, directory} -> ok; {value, _} -> - report_error("`~s' is not a directory.", + report_error("`~ts' is not a directory.", [filename(Dir)]), exit(error); none -> case file:make_dir(Dir) of ok -> - verbose("created directory `~s'.", + verbose("created directory `~ts'.", [filename(Dir)], Opts), ok; E -> report_error("failed to create " - "directory `~s'.", + "directory `~ts'.", [filename(Dir)]), exit({make_dir, E}) end @@ -385,7 +385,7 @@ write_module(Tree, Name, Opts) -> end, Printer = proplists:get_value(printer, Opts), FD = open_output_file(File, Encoding), - verbose("writing to file `~s'.", [File], Opts), + verbose("writing to file `~ts'.", [File], Opts), V = (catch {ok, output(FD, Printer, Tree, Opts++Encoding)}), ok = file:close(FD), case V of @@ -471,7 +471,7 @@ backup_file_1(Name, Opts) -> filename:basename(Name) ++ Suffix), case catch file:rename(Name, Dest) of ok -> - verbose("made backup of file `~s'.", [Name], Opts); + verbose("made backup of file `~ts'.", [Name], Opts); {error, R} -> error_backup_file(Name), exit({error, R}); @@ -1839,17 +1839,17 @@ report_export_vars(F, L, Type, Opts) -> [Type], Opts). error_read_file(Name) -> - report_error("error reading file `~s'.", [filename(Name)]). + report_error("error reading file `~ts'.", [filename(Name)]). error_write_file(Name) -> - report_error("error writing to file `~s'.", [filename(Name)]). + report_error("error writing to file `~ts'.", [filename(Name)]). error_backup_file(Name) -> - report_error("could not create backup of file `~s'.", + report_error("could not create backup of file `~ts'.", [filename(Name)]). error_open_output(Name) -> - report_error("cannot open file `~s' for output.", [filename(Name)]). + report_error("cannot open file `~ts' for output.", [filename(Name)]). verbosity(Opts) -> case proplists:get_bool(quiet, Opts) of @@ -1908,9 +1908,9 @@ format({"", L, D}, Vs) when is_integer(L), L > 0 -> format({"", _L, D}, Vs) -> format(D, Vs); format({F, L, D}, Vs) when is_integer(L), L > 0 -> - [io_lib:fwrite("~s:~w: ", [filename(F), L]), format(D, Vs)]; + [io_lib:fwrite("~ts:~w: ", [filename(F), L]), format(D, Vs)]; format({F, _L, D}, Vs) -> - [io_lib:fwrite("~s: ", [filename(F)]), format(D, Vs)]; + [io_lib:fwrite("~ts: ", [filename(F)]), format(D, Vs)]; format(S, Vs) when is_list(S) -> [io_lib:fwrite(S, Vs), $\n]. diff --git a/lib/syntax_tools/src/igor.erl b/lib/syntax_tools/src/igor.erl index 567bec00cb..d385c2b690 100644 --- a/lib/syntax_tools/src/igor.erl +++ b/lib/syntax_tools/src/igor.erl @@ -2749,7 +2749,7 @@ read_module(Name, Options) -> end. read_module_1(Name, Options) -> - verbose("reading module `~s'.", [filename(Name)], Options), + verbose("reading module `~ts'.", [filename(Name)], Options), {Forms, Enc} = read_module_2(Name, Options), case proplists:get_bool(comments, Options) of false -> @@ -2794,7 +2794,7 @@ check_forms([F | Fs], File) -> _ -> "unknown error" end, - report_error("in file `~s' at line ~w:\n ~ts", + report_error("in file `~ts' at line ~w:\n ~ts", [filename(File), erl_syntax:get_pos(F), S]), exit(error); _ -> @@ -2841,18 +2841,18 @@ write_module(Tree, Name, Dir, Opts) -> {value, directory} -> ok; {value, _} -> - report_error("`~s' is not a directory.", + report_error("`~ts' is not a directory.", [Dir1]), exit(error); none -> case file:make_dir(Dir1) of ok -> - verbose("created directory `~s'.", + verbose("created directory `~ts'.", [Dir1], Opts), ok; E -> report_error("failed to create " - "directory `~s'.", + "directory `~ts'.", [Dir1]), exit({make_dir, E}) end @@ -2870,7 +2870,7 @@ write_module(Tree, Name, Dir, Opts) -> Printer = proplists:get_value(printer, Opts), FD = open_output_file(File), ok = output_encoding(FD, Opts), - verbose("writing to file `~s'.", [File], Opts), + verbose("writing to file `~ts'.", [File], Opts), V = (catch {ok, output(FD, Printer, Tree, Opts)}), ok = file:close(FD), case V of @@ -2911,7 +2911,7 @@ backup_file_1(Name, Opts) -> filename:basename(Name1) ++ Suffix), case catch file:rename(Name1, Dest) of ok -> - verbose("made backup of file `~s'.", [Name1], Opts); + verbose("made backup of file `~ts'.", [Name1], Opts); {error, R} -> error_backup_file(Name1), exit({error, R}); @@ -3036,19 +3036,19 @@ warning_apply_2(Module, Target) -> "possibly unsafe in `~s'.", [Module, Target]). error_open_output(Name) -> - report_error("cannot open file `~s' for output.", [filename(Name)]). + report_error("cannot open file `~ts' for output.", [filename(Name)]). error_read_file(Name) -> - report_error("error reading file `~s'.", [filename(Name)]). + report_error("error reading file `~ts'.", [filename(Name)]). error_read_file_info(Name) -> - report_error("error getting file info: `~s'.", [filename(Name)]). + report_error("error getting file info: `~ts'.", [filename(Name)]). error_write_file(Name) -> - report_error("error writing to file `~s'.", [filename(Name)]). + report_error("error writing to file `~ts'.", [filename(Name)]). error_backup_file(Name) -> - report_error("could not create backup of file `~s'.", + report_error("could not create backup of file `~ts'.", [filename(Name)]). verbose(S, Opts) -> diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index 7fcd22e455..01569687bf 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 1.6.10 +SYNTAX_TOOLS_VSN = 1.6.11 diff --git a/lib/test_server/doc/src/notes.xml b/lib/test_server/doc/src/notes.xml index 59fb789e35..b35929f1e6 100644 --- a/lib/test_server/doc/src/notes.xml +++ b/lib/test_server/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -32,6 +32,62 @@ <file>notes.xml</file> </header> +<section><title>Test_Server 3.6.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The unicode update of test_server for R16A introduced a + few potential errors when logging to files. Sometimes ~tp + or ~ts was used for formatting also when writing to files + that were not opened with the {encoding,utf8} option. If + then the argument contained unicode characters above 255, + the file descriptor would crash. This has been corrected + by the following modifications: <list> <item> Since the + 'unexpected_io' log file is used only when the test case + HTML file is not available (e.g. between test cases), + this file is now also a HTML file and as other + test_server HTML logs it is always UTF-8 encoded </item> + <item> Since it is possible to change which information + is going to which log file (with + test_server_ctrl:set_levels/3), we do not have full + control over which information is written to which file. + This means that any printout could be written to the + 'major' log file (suite.log), which was earlier encoded + as latin1. To avoid crashing this file descriptor due to + unicode strings, the 'major' log file is now also encoded + in UTF-8 (possible incopatibility). </item> <item> The + cross_cover.info file is no longer a text file which can + be read with file:consult/1, instead it is written as a + pure binary file using term_to_binary when writing and + binary_to_term when reading. </item> <item> The encoding + of the file named 'last_name', which only content is the + path to the last run.<timestamp> directory, is now + dependent on the file name mode of the VM. If file names + are expected to be unicode, then the 'last_name' file is + UTF-8 encoded, else it is latin1 encoded. </item> </list></p> + <p> + Also, ~tp has been changed back to ~p unless it is + somehow likely that the argument includes strings. It is + not obvious that this is the correct thing to do, but + some decission had to be taken...</p> + <p> + Own Id: OTP-10780</p> + </item> + <item> + <p> + Using the force_stop flag/option to interrupt a test run + caused a crash in Common Test. This problem has been + solved.</p> + <p> + Own Id: OTP-10832</p> + </item> + </list> + </section> + +</section> + <section><title>Test_Server 3.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 70cb6fa220..5d4d392166 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -681,7 +681,7 @@ handle_call({abort_current_testcase,Reason}, _From, State) -> handle_call({finish,Fini}, _From, State) -> case State#state.jobs of [] -> - lists:foreach(fun({Cli,Fun}) -> Fun(Cli) end, + lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Fini) end, State#state.idle_notify), State2 = State#state{finish=false}, {stop,shutdown,{ok,self()}, State2}; @@ -699,14 +699,11 @@ handle_call({finish,Fini}, _From, State) -> handle_call({idle_notify,Fun}, {Cli,_Ref}, State) -> case State#state.jobs of - [] -> - Fun(Cli), - {reply, {ok,self()}, State}; - _ -> - Subscribed = State#state.idle_notify, - {reply, {ok,self()}, - State#state{idle_notify=[{Cli,Fun}|Subscribed]}} - end; + [] -> self() ! report_idle; + _ -> ok + end, + Subscribed = State#state.idle_notify, + {reply, {ok,self()}, State#state{idle_notify=[{Cli,Fun}|Subscribed]}}; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% handle_call(start_get_totals, From, State) -> {ok,Pid} @@ -1000,6 +997,13 @@ handle_cast({node_started,Node}, State) -> %% lost contact with target. The test_server_ctrl process is %% terminated, and teminate/2 will do the cleanup +handle_info(report_idle, State) -> + Finish = State#state.finish, + lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end, + State#state.idle_notify), + {noreply,State#state{idle_notify=[]}}; + + handle_info({'EXIT',Pid,Reason}, State) -> case lists:keysearch(Pid,2,State#state.jobs) of false -> @@ -1017,11 +1021,12 @@ handle_info({'EXIT',Pid,Reason}, State) -> [Name,Reason]) end, State2 = State#state{jobs=NewJobs}, + Finish = State2#state.finish, case NewJobs of [] -> - lists:foreach(fun({Cli,Fun}) -> Fun(Cli) end, + lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end, State2#state.idle_notify), - case State2#state.finish of + case Finish of false -> {noreply,State2#state{idle_notify=[]}}; _ -> % true | abort @@ -1031,9 +1036,9 @@ handle_info({'EXIT',Pid,Reason}, State) -> {stop,shutdown,State2#state{finish=false}} end; _ -> % pending jobs - case State2#state.finish of + case Finish of abort -> % abort test now! - lists:foreach(fun({Cli,Fun}) -> Fun(Cli) end, + lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end, State2#state.idle_notify), {stop,shutdown,State2#state{finish=false}}; _ -> % true | false diff --git a/lib/test_server/src/ts_run.erl b/lib/test_server/src/ts_run.erl index 741dd483f5..60c9a7a4b7 100644 --- a/lib/test_server/src/ts_run.erl +++ b/lib/test_server/src/ts_run.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -261,13 +261,17 @@ run_batch(Vars, _Spec, State) -> ts_lib:progress(Vars, 1, "Command: ~s~n", [Command]), io:format(user, "Command: ~s~n",[Command]), Port = open_port({spawn, Command}, [stream, in, eof]), - tricky_print_data(Port). + Timeout = 30000 * case os:getenv("TS_RUN_VALGRIND") of + false -> 1; + _ -> 100 + end, + tricky_print_data(Port, Timeout). -tricky_print_data(Port) -> +tricky_print_data(Port, Timeout) -> receive {Port, {data, Bytes}} -> io:put_chars(Bytes), - tricky_print_data(Port); + tricky_print_data(Port, Timeout); {Port, eof} -> Port ! {self(), close}, receive @@ -280,7 +284,7 @@ tricky_print_data(Port) -> after 1 -> % force context switch ok end - after 30000 -> + after Timeout -> case erl_epmd:names() of {ok,Names} -> case is_testnode_dead(Names) of @@ -288,10 +292,10 @@ tricky_print_data(Port) -> io:put_chars("WARNING: No EOF, but " "test_server node is down!\n"); false -> - tricky_print_data(Port) + tricky_print_data(Port, Timeout) end; _ -> - tricky_print_data(Port) + tricky_print_data(Port, Timeout) end end. diff --git a/lib/test_server/vsn.mk b/lib/test_server/vsn.mk index 8949f13b2f..0a5c4246f1 100644 --- a/lib/test_server/vsn.mk +++ b/lib/test_server/vsn.mk @@ -1 +1 @@ -TEST_SERVER_VSN = 3.6 +TEST_SERVER_VSN = 3.6.1 diff --git a/lib/toolbar/doc/src/notes.xml b/lib/toolbar/doc/src/notes.xml index 925c9c259c..0ae8e261d6 100644 --- a/lib/toolbar/doc/src/notes.xml +++ b/lib/toolbar/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/tools/doc/src/cover.xml b/lib/tools/doc/src/cover.xml index a2444ec947..beefd4ee8d 100644 --- a/lib/tools/doc/src/cover.xml +++ b/lib/tools/doc/src/cover.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2001</year> - <year>2012</year> + <year>2013</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -214,7 +214,9 @@ <c>{no_abstract_code,BeamFile}</c> is returned. If the abstract code is encrypted, and no key is available for decrypting it, the error reason - <c><![CDATA[{encrypted_abstract_code,BeamFile} is returned. <p>If only the module name (i.e. not the full name of the <c>.beam]]></c> file) is given to this function, the + <c>{encrypted_abstract_code,BeamFile}</c> is returned.</p> + <p>If only the module name (i.e. not the full name of the + <c>.beam</c> file) is given to this function, the <c>.beam</c> file is found by calling <c>code:which(Module)</c>. If no <c>.beam</c> file is found, the error reason <c>non_existing</c> is returned. If the @@ -313,9 +315,15 @@ file, i.e. using <c>compile_beam/1</c> or <c>compile_beam_directory/0,1</c>, it is assumed that the source code can be found in the same directory as the - <c>.beam</c> file, or in <c>../src</c> relative to that - directory. If no source code is found, - <c>,{error,no_source_code_found}</c> is returned.</p> + <c>.beam</c> file, in <c>../src</c> relative to that + directory, or using the source path in + <c>Module:module_info(compile)</c>. When using the latter, + two paths are examined: first the one constructed by + joining <c>../src</c> and the tail of the compiled path + below a trailing <c>src</c> component, then the compiled + path itself. + If no source code is found, + <c>{error,no_source_code_found}</c> is returned.</p> <p>HINT: It is possible to issue multiple analyse_to_file commands at the same time. </p> </desc> diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index 7f2fe34666..20dc9679f1 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -30,6 +30,22 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 2.6.10</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fix a bug in cover when used with no_auto_import. Thanks + to Jos� Valim.</p> + <p> + Own Id: OTP-10778</p> + </item> + </list> + </section> + +</section> + <section><title>Tools 2.6.9</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl index 2579711dc7..dfcfc3675f 100644 --- a/lib/tools/src/cover.erl +++ b/lib/tools/src/cover.erl @@ -1951,47 +1951,61 @@ move_clauses([{M,F,A,C,_L}|Clauses]) -> move_clauses(Clauses); move_clauses([]) -> ok. - %% Given a .beam file, find the .erl file. Look first in same directory as -%% the .beam file, then in <beamdir>/../src +%% the .beam file, then in ../src, then in compile info. find_source(Module, File0) -> - case filename:rootname(File0,".beam") of - File0 -> - File0; - File -> - InSameDir = File++".erl", - case filelib:is_file(InSameDir) of - true -> - InSameDir; - false -> - Dir = filename:dirname(File), - Mod = filename:basename(File), - InDotDotSrc = filename:join([Dir,"..","src",Mod++".erl"]), - case filelib:is_file(InDotDotSrc) of - true -> - InDotDotSrc; - false -> - find_source_from_module(Module, File0) - end - end + try + Root = filename:rootname(File0, ".beam"), + Root == File0 andalso throw(File0), %% not .beam + %% Look for .erl in pwd. + File = Root ++ ".erl", + throw_file(File), + %% Not in pwd: look in ../src. + BeamDir = filename:dirname(File), + Base = filename:basename(File), + throw_file(filename:join([BeamDir, "..", "src", Base])), + %% Not in ../src: look for source path in compile info, but + %% first look relative the beam directory. + Info = lists:keyfind(source, 1, Module:module_info(compile)), + false == Info andalso throw({beam, File0}), %% stripped + {source, SrcFile} = Info, + throw_file(splice(BeamDir, SrcFile)), %% below ../src + throw_file(SrcFile), %% or absolute + %% No success means that source is either not under ../src or + %% its relative path differs from that of compile info. (For + %% example, compiled under src/x but installed under src/y.) + %% An option to specify an arbitrary source path explicitly is + %% probably a better solution than either more heuristics or a + %% potentially slow filesystem search. + {beam, File0} + catch + Path -> Path end. -%% In case we can't find the file from the given .beam, -%% we try to get the information directly from the module source -find_source_from_module(Module, File) -> - Compile = Module:module_info(compile), - case lists:keyfind(source, 1, Compile) of - {source, Path} -> - case filelib:is_file(Path) of - true -> - Path; - false -> - {beam, File} - end; - false -> - {beam, File} - end. +throw_file(Path) -> + false /= Path andalso filelib:is_file(Path) andalso throw(Path). + +%% Splice the tail of a source path, starting from the last "src" +%% component, onto the parent of a beam directory, or return false if +%% no "src" component is found. +%% +%% Eg. splice("/path/to/app-1.0/ebin", "/compiled/path/to/app/src/x/y.erl") +%% --> "/path/to/app-1.0/ebin/../src/x/y.erl" +%% +%% This handles the case of source in subdirectories of ../src with +%% beams that have moved since compilation. +%% +splice(BeamDir, SrcFile) -> + case lists:splitwith(fun(C) -> C /= "src" end, revsplit(SrcFile)) of + {T, [_|_]} -> %% found src component + filename:join([BeamDir, "..", "src" | lists:reverse(T)]); + {_, []} -> %% or not + false + end. + +revsplit(Path) -> + lists:reverse(filename:split(Path)). do_parallel_analysis(Module, Analysis, Level, Loaded, From, State) -> analyse_info(Module,State#main_state.imported), diff --git a/lib/tv/doc/src/notes.xml b/lib/tv/doc/src/notes.xml index a80f8080b0..c95171568f 100644 --- a/lib/tv/doc/src/notes.xml +++ b/lib/tv/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/webtool/doc/src/notes.xml b/lib/webtool/doc/src/notes.xml index c5fd340b50..76b73daa59 100644 --- a/lib/webtool/doc/src/notes.xml +++ b/lib/webtool/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2011</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/wx/doc/src/notes.xml b/lib/wx/doc/src/notes.xml index 87f73be9ed..a6e89d32a9 100644 --- a/lib/wx/doc/src/notes.xml +++ b/lib/wx/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2009</year><year>2012</year> + <year>2009</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/xmerl/doc/src/notes.xml b/lib/xmerl/doc/src/notes.xml index 4f6a917de5..1de854860c 100644 --- a/lib/xmerl/doc/src/notes.xml +++ b/lib/xmerl/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> |
