diff options
Diffstat (limited to 'lib')
272 files changed, 10495 insertions, 2986 deletions
diff --git a/lib/asn1/test/asn1_SUITE_data/DirectoryAbstractService.asn b/lib/asn1/test/asn1_SUITE_data/DirectoryAbstractService.asn index 5a5d310729..5a5d310729 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/DirectoryAbstractService.asn +++ b/lib/asn1/test/asn1_SUITE_data/DirectoryAbstractService.asn diff --git a/lib/asn1/test/asn1_SUITE_data/InformationFramework.asn b/lib/asn1/test/asn1_SUITE_data/InformationFramework.asn index ef236e98a9..ef236e98a9 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/InformationFramework.asn +++ b/lib/asn1/test/asn1_SUITE_data/InformationFramework.asn diff --git a/lib/asn1/test/asn1_SUITE_data/LDAP.asn1 b/lib/asn1/test/asn1_SUITE_data/LDAP.asn1 index 4d845942e1..4d845942e1 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/LDAP.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/LDAP.asn1 diff --git a/lib/asn1/test/asn1_SUITE_data/Nortel.asn b/lib/asn1/test/asn1_SUITE_data/Nortel.asn index a27c78a0b5..a27c78a0b5 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/Nortel.asn +++ b/lib/asn1/test/asn1_SUITE_data/Nortel.asn diff --git a/lib/asn1/test/asn1_SUITE_data/UpperBounds.asn b/lib/asn1/test/asn1_SUITE_data/UpperBounds.asn index 247260495b..247260495b 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/UpperBounds.asn +++ b/lib/asn1/test/asn1_SUITE_data/UpperBounds.asn diff --git a/lib/asn1/test/asn1_SUITE_data/UsefulDefinitions.asn b/lib/asn1/test/asn1_SUITE_data/UsefulDefinitions.asn index d9601bb7d0..d9601bb7d0 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/UsefulDefinitions.asn +++ b/lib/asn1/test/asn1_SUITE_data/UsefulDefinitions.asn diff --git a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-CommonDataTypes.asn b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-CommonDataTypes.asn index e3f6e83d6b..e3f6e83d6b 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-CommonDataTypes.asn +++ b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-CommonDataTypes.asn diff --git a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Constants.asn b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Constants.asn index 1411d455b7..1411d455b7 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Constants.asn +++ b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Constants.asn diff --git a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Containers.asn b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Containers.asn index fb08451103..fb08451103 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Containers.asn +++ b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-Containers.asn diff --git a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-IEs.asn b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-IEs.asn index 848d8f6099..848d8f6099 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-IEs.asn +++ b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-IEs.asn diff --git a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Contents.asn b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Contents.asn index 9ecfa688a2..9ecfa688a2 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Contents.asn +++ b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Contents.asn diff --git a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Discriptions.asn b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Discriptions.asn index b9be9934e4..b9be9934e4 100755..100644 --- a/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Discriptions.asn +++ b/lib/asn1/test/asn1_SUITE_data/nbapsystem/NBAP-PDU-Discriptions.asn diff --git a/lib/common_test/doc/src/filestruct.gif b/lib/common_test/doc/src/filestruct.gif Binary files differindex 2b06833d0b..2b06833d0b 100755..100644 --- a/lib/common_test/doc/src/filestruct.gif +++ b/lib/common_test/doc/src/filestruct.gif diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 10fe333d18..802c1991de 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -154,7 +154,7 @@ static ERL_NIF_TERM exor(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rc4_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM rc2_40_cbc_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM rc2_cbc_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dss_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rsa_public_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -234,7 +234,7 @@ static ErlNifFunc nif_funcs[] = { {"rc4_encrypt", 2, rc4_encrypt}, {"rc4_set_key", 1, rc4_set_key}, {"rc4_encrypt_with_state", 2, rc4_encrypt_with_state}, - {"rc2_40_cbc_crypt", 4, rc2_40_cbc_crypt}, + {"rc2_cbc_crypt", 4, rc2_cbc_crypt}, {"rsa_sign_nif", 3, rsa_sign_nif}, {"dss_sign_nif", 3, dss_sign_nif}, {"rsa_public_crypt", 4, rsa_public_crypt}, @@ -1237,30 +1237,31 @@ static ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_N return enif_make_tuple2(env,new_state,new_data); } -static ERL_NIF_TERM rc2_40_cbc_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static ERL_NIF_TERM rc2_cbc_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Key,IVec,Data,IsEncrypt) */ ErlNifBinary key_bin, ivec_bin, data_bin; RC2_KEY rc2_key; ERL_NIF_TERM ret; - + unsigned char iv_copy[8]; + if (!enif_inspect_iolist_as_binary(env, argv[0], &key_bin) - || key_bin.size != 5 + || (key_bin.size != 5 && key_bin.size != 8 && key_bin.size != 16) || !enif_inspect_binary(env, argv[1], &ivec_bin) || ivec_bin.size != 8 - || !enif_inspect_iolist_as_binary(env, argv[2], &data_bin)) { - + || !enif_inspect_iolist_as_binary(env, argv[2], &data_bin) + || data_bin.size % 8 != 0) { return enif_make_badarg(env); } - - RC2_set_key(&rc2_key, 5, key_bin.data, 40); + + RC2_set_key(&rc2_key, key_bin.size, key_bin.data, key_bin.size*8); + memcpy(iv_copy, ivec_bin.data, 8); RC2_cbc_encrypt(data_bin.data, - enif_make_new_binary(env, data_bin.size, &ret), + enif_make_new_binary(env, data_bin.size, &ret), data_bin.size, &rc2_key, - ivec_bin.data, + iv_copy, (argv[3] == atom_true)); - return ret; -} +} static ERL_NIF_TERM rsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Type,Data,Key=[E,N,D]) */ diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 824be09438..48243fd693 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="iso-8859-1" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> @@ -334,14 +334,16 @@ Mpint() = <![CDATA[<<ByteLen:32/integer-big, Bytes:ByteLen/binary>>]]> </func> <func> <name>sha_mac(Key, Data) -> Mac</name> + <name>sha_mac(Key, Data, MacLength) -> Mac</name> <fsummary>Compute an <c>MD5 MAC</c>message authentification code</fsummary> <type> <v>Key = Data = iolist() | binary()</v> <v>Mac = binary()</v> + <v>MacLenength = integer() =< 20 </v> </type> <desc> <p>Computes an <c>SHA MAC</c> message authentification code - from <c>Key</c> and <c>Data</c>, where the length of the Mac + from <c>Key</c> and <c>Data</c>, where the default length of the Mac is 160 bits (20 bytes).</p> </desc> </func> @@ -1046,6 +1048,30 @@ Mpint() = <![CDATA[<<ByteLen:32/integer-big, Bytes:ByteLen/binary>>]]> </func> <func> + <name>rc2_cbc_encrypt(Key, IVec, Text) -> Cipher</name> + <fsummary>Encrypt <c>Text</c>according to RC2 in CBC mode</fsummary> + <type> + <v>Key = Text = iolist() | binary()</v> + <v>Ivec = Cipher = binary()</v> + </type> + <desc> + <p>Encrypts <c>Text</c> according to RC2 in CBC mode.</p> + </desc> + </func> + + <func> + <name>rc2_cbc_decrypt(Key, IVec, Cipher) -> Text</name> + <fsummary>Decrypts <c>Cipher</c>according to RC2 in CBC mode</fsummary> + <type> + <v>Key = Text = iolist() | binary()</v> + <v>Ivec = Cipher = binary()</v> + </type> + <desc> + <p>Decrypts <c>Cipher</c> according to RC2 in CBC mode.</p> + </desc> + </func> + + <func> <name>rc4_encrypt(Key, Data) -> Result</name> <fsummary>Encrypt data using RC4</fsummary> <type> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index e3b921f9fa..0714cb686d 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -27,7 +27,7 @@ -export([sha/1, sha_init/0, sha_update/2, sha_final/1]). %-export([sha256/1, sha256_init/0, sha256_update/2, sha256_final/1]). %-export([sha512/1, sha512_init/0, sha512_update/2, sha512_final/1]). --export([md5_mac/2, md5_mac_96/2, sha_mac/2, sha_mac_96/2]). +-export([md5_mac/2, md5_mac_96/2, sha_mac/2, sha_mac/3, sha_mac_96/2]). -export([hmac_init/2, hmac_update/2, hmac_final/1, hmac_final_n/2]). -export([des_cbc_encrypt/3, des_cbc_decrypt/3, des_cbc_ivec/1]). -export([des_ecb_encrypt/2, des_ecb_decrypt/2]). @@ -42,7 +42,7 @@ -export([aes_cfb_128_encrypt/3, aes_cfb_128_decrypt/3]). -export([exor/2]). -export([rc4_encrypt/2, rc4_set_key/1, rc4_encrypt_with_state/2]). --export([rc2_40_cbc_encrypt/3, rc2_40_cbc_decrypt/3]). +-export([rc2_cbc_encrypt/3, rc2_cbc_decrypt/3, rc2_40_cbc_encrypt/3, rc2_40_cbc_decrypt/3]). -export([dss_verify/3, dss_verify/4, rsa_verify/3, rsa_verify/4]). -export([dss_sign/2, dss_sign/3, rsa_sign/2, rsa_sign/3]). -export([rsa_public_encrypt/3, rsa_private_decrypt/3]). @@ -83,7 +83,7 @@ dss_verify,dss_sign, rsa_verify,rsa_sign, rsa_public_encrypt,rsa_private_decrypt, - rsa_private_encrypt,rsa_public_decrypt, + rsa_private_encrypt,rsa_public_decrypt, dh_generate_key, dh_compute_key, aes_cbc_128_encrypt, aes_cbc_128_decrypt, exor, @@ -91,7 +91,7 @@ rc2_40_cbc_encrypt, rc2_40_cbc_decrypt, %% idea_cbc_encrypt, idea_cbc_decrypt, aes_cbc_256_encrypt, aes_cbc_256_decrypt, - aes_ctr_encrypt, aes_ctr_decrypt, + aes_ctr_encrypt, aes_ctr_decrypt, aes_ctr_stream_init, aes_ctr_stream_encrypt, aes_ctr_stream_decrypt, info_lib]). @@ -260,6 +260,9 @@ md5_mac_n(_Key,_Data,_MacSz) -> ?nif_stub. sha_mac(Key, Data) -> sha_mac_n(Key,Data,20). +sha_mac(Key, Data, Size) -> + sha_mac_n(Key, Data, Size). + sha_mac_96(Key, Data) -> sha_mac_n(Key,Data,12). @@ -689,16 +692,25 @@ rc4_encrypt(_Key, _Data) -> ?nif_stub. rc4_set_key(_Key) -> ?nif_stub. rc4_encrypt_with_state(_State, _Data) -> ?nif_stub. + +%% RC2 block cipher + +rc2_cbc_encrypt(Key, IVec, Data) -> + rc2_cbc_crypt(Key,IVec,Data,true). + +rc2_cbc_decrypt(Key, IVec, Data) -> + rc2_cbc_crypt(Key,IVec,Data,false). + +rc2_cbc_crypt(_Key, _IVec, _Data, _IsEncrypt) -> ?nif_stub. + %% -%% RC2 - 40 bits block cipher +%% RC2 - 40 bits block cipher - Backwards compatibility not documented. %% -rc2_40_cbc_encrypt(Key, IVec, Data) -> - rc2_40_cbc_crypt(Key,IVec,Data,true). - -rc2_40_cbc_decrypt(Key, IVec, Data) -> - rc2_40_cbc_crypt(Key,IVec,Data,false). +rc2_40_cbc_encrypt(Key, IVec, Data) when erlang:byte_size(Key) == 5 -> + rc2_cbc_crypt(Key,IVec,Data,true). -rc2_40_cbc_crypt(_Key, _IVec, _Data, _IsEncrypt) -> ?nif_stub. +rc2_40_cbc_decrypt(Key, IVec, Data) when erlang:byte_size(Key) == 5 -> + rc2_cbc_crypt(Key,IVec,Data,false). %% %% DH Diffie-Hellman functions diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 53b4c2a7e1..86acdc27df 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -49,6 +49,7 @@ des_ecb/1, des3_cbc/1, des3_cfb/1, + rc2_cbc/1, aes_cfb/1, aes_cbc/1, aes_cbc_iter/1, @@ -79,8 +80,10 @@ all() -> md5_mac_io, sha, sha_update, hmac_update_sha, hmac_update_sha_n, hmac_update_md5_n, hmac_update_md5_io, hmac_update_md5, %% sha256, sha256_update, sha512,sha512_update, - des_cbc, des_cfb, des3_cbc, des3_cfb, aes_cfb, aes_cbc, + des_cbc, des_cfb, des3_cbc, des3_cfb, rc2_cbc, aes_cfb, aes_cbc, aes_cbc_iter, aes_ctr, aes_ctr_stream, des_cbc_iter, des_cfb_iter, des_ecb, + des_cbc, rc2_cbc, aes_cfb, aes_cbc, + aes_cbc_iter, aes_ctr, aes_ctr_stream, des_cbc_iter, des_ecb, rand_uniform_test, strong_rand_test, rsa_verify_test, dsa_verify_test, rsa_sign_test, dsa_sign_test, rsa_encrypt_decrypt, dh, exor_test, @@ -347,7 +350,7 @@ hmac_update_md5(Config) when is_list(Config) -> Key2 = "A fine speach by a fine man!", ?line Long1 = "Four score and seven years ago our fathers brought forth on this continent a new nation, conceived in liberty, and dedicated to the proposition that all men are created equal.", ?line Long2 = "Now we are engaged in a great civil war, testing whether that nation, or any nation, so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this.", - ?line Long3 = "But, in a larger sense, we can not dedicate, we can not consecrate, we can not hallow this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us-that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion—that we here highly resolve that these dead shall not have died in vain-that this nation, under God, shall have a new birth of freedom-and that government of the people, by the people, for the people, shall not perish from the earth.", + ?line Long3 = "But, in a larger sense, we can not dedicate, we can not consecrate, we can not hallow this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us-that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion that we here highly resolve that these dead shall not have died in vain-that this nation, under God, shall have a new birth of freedom-and that government of the people, by the people, for the people, shall not perish from the earth.", ?line CtxA = crypto:hmac_init(md5, Key2), ?line CtxB = crypto:hmac_update(CtxA, Long1), ?line CtxC = crypto:hmac_update(CtxB, Long2), @@ -604,6 +607,21 @@ des_ecb(Config) when is_list(Config) -> ?line m(Cipher5, <<"he time ">>), ?line Cipher6 = crypto:des_ecb_decrypt(Key, hexstr2bin("893d51ec4b563b53")), ?line m(Cipher6, <<"for all ">>). +%% +%% +rc2_cbc(doc) -> + "Encrypt and decrypt according to RC2 CBC and check the result. " + "Example stripped out from public_key application test"; +rc2_cbc(Config) when is_list(Config) -> + + Key = <<146,210,160,124,215,227,153,239,227,17,222,140,3,93,27,191>>, + IV = <<72,91,135,182,25,42,35,210>>, + + Cipher = <<36,245,206,158,168,230,58,69,148,137,32,192,250,41,237,181,181,251, 192,2,175,135,177,171,57,30,111,117,159,149,15,28,88,158,28,81,28,115, 85,219,241,82,117,222,91,85,73,117,164,25,182,52,191,64,123,57,26,19, 211,27,253,31,194,219,231,104,247,240,172,130,119,21,225,154,101,247, 32,216,42,216,133,169,78,22,97,27,227,26,196,224,172,168,17,9,148,55, 203,91,252,40,61,226,236,221,215,160,78,63,13,181,68,57,196,241,185, 207, 116,129,152,237,60,139,247,153,27,146,161,246,222,98,185,222,152, 187,135, 236,86,34,7,110,91,230,173,34,160,242,202,222,121,127,181,140, 101,203,195, 190,88,250,86,147,127,87,72,126,171,16,71,47,110,248,88, 14,29,143,161,152, 129,236,148,22,152,186,208,119,70,8,174,193,203,100, 193,203,200,117,102,242, 134,142,96,125,135,200,217,190,76,117,50,70, 209,186,101,241,200,91,40,193,54, 90,195,38,47,59,197,38,234,86,223,16, 51,253,204,129,20,171,66,21,241,26,135,216, 196,114,110,91,15,53,40, 164,201,136,113,95,247,51,181,208,241,68,168,98,151,36, 155,72,24,57, 42,191,14,125,204,10,167,214,233,138,115,125,234,121,134,227,26,247, 77,200,117,110,117,111,168,156,206,67,159,149,189,173,150,193,91,199, 216,153,22, 189,137,185,89,160,13,131,132,58,109,28,110,246,252,251,14, 232,91,38,52,29,101,188,69,123,50,0,130,178,93,73,239,118,7,77,35,59, 253,10,159,45,86,142,37,78,232,48>>, + Text = <<48,130,1,85,2,1,0,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,4,130,1,63,48,130, 1,59,2,1,0,2,65,0,222,187,252,44,9,214,27,173,162,169,70,47,36,34,78,84,204, 107,60,192,117,95,21,206,49,142,245,126,121,223,23,2,107,106,133,204,161,36, 40,2,114,69,4,93,242,5,42,50,154,47,154,211,209,123,120,161,5,114,173,155,34, 191,52,59,2,3,1,0,1,2,64,45,144,169,106,220,236,71,39,67,82,123,192,35,21,61, 143,13,110,150,180,12,142,210,40,39,109,70,125,132,51,6,66,159,134,112,85, 155,243,118,221,65,133,127,99,151,194,252,141,149,224,229,62,214,45,228,32, 184,85,67,14,228,161,184,161,2,33,0,255,202,240,131,130,57,49,224,115,255,83, 79,6,165,212,21,179,212,20,188,97,74,69,68,163,223,247,237,39,24,23,235,2,33, 0,222,234,48,36,33,23,219,45,59,136,55,245,143,29,165,48,255,131,207,146,131, 104,13,163,54,131,236,78,88,54,16,241,2,33,0,230,2,99,129,173,176,166,131, 241,106,143,76,9,107,70,41,121,185,228,39,124,200,159,62,216,169,5,180,111, 169,255,159,2,33,0,151,193,70,212,209,210,179,219,175,83,165,4,255,81,103,76, 92,39,24,0,222,132,208,3,244,241,10,198,171,54,227,129,2,32,43,250,20,31,16, 189,168,116,225,1,125,132,94,130,118,124,28,56,232,39,69,218,244,33,240,200, 205,9,215,101,35,135,7,7,7,7,7,7,7>>, + + Text = crypto:rc2_cbc_decrypt(Key, IV, Cipher), + Cipher = crypto:rc2_cbc_encrypt(Key, IV, Text). %% %% diff --git a/lib/debugger/src/dbg_ui_break_win.erl b/lib/debugger/src/dbg_ui_break_win.erl index 4039bf785f..abbec158b0 100644 --- a/lib/debugger/src/dbg_ui_break_win.erl +++ b/lib/debugger/src/dbg_ui_break_win.erl @@ -81,12 +81,12 @@ create_win(GS, {X, Y}, function, Mod, _Line) -> {pack_x, 2}, {pack_y, 3}, {selectmode, multiple}]), - %% Add Ok and Cancel buttons - {Wbtn, Hbtn} = dbg_ui_win:min_size(["Ok","Cancel"], 70, 30), + %% Add OK and Cancel buttons + {Wbtn, Hbtn} = dbg_ui_win:min_size(["OK","Cancel"], 70, 30), Bot = gs:frame(Frm, [{pack_x, {1, 3}}, {pack_y, 4}]), - Ok = gs:button(Bot, [{x, Pad}, {y, Pad}, + OK = gs:button(Bot, [{x, Pad}, {y, Pad}, {width, Wbtn}, {height, Hbtn}, - {label, {text,"Ok"}}, {font, Font}]), + {label, {text,"OK"}}, {font, Font}]), Cancel = gs:button(Bot, [{x, W-Pad-Wbtn}, {y, Pad}, {width, Wbtn}, {height, Hbtn}, {label, {text,"Cancel"}}, {font, Font}]), @@ -95,7 +95,7 @@ create_win(GS, {X, Y}, function, Mod, _Line) -> gs:config(Win, [{width, Wfrm}, {height, Hfrm}, {map, true}]), #winInfo{type=function, win=Win, packer=Frm, entries=Entries, trigger=enable, - ok=Ok, cancel=Cancel, listbox=Lb, funcs=[]}; + ok=OK, cancel=Cancel, listbox=Lb, funcs=[]}; create_win(GS, {X, Y}, Type, Mod, Line) -> Pad = 8, W = 230, @@ -161,12 +161,12 @@ create_win(GS, {X, Y}, Type, Mod, Line) -> {align, w}, {group, Grp}, {data, {trigger, delete}}]), - %% Add Ok and Cancel buttons - {Wbtn, Hbtn} = dbg_ui_win:min_size(["Ok","Cancel"], 70, 30), + %% Add OK and Cancel buttons + {Wbtn, Hbtn} = dbg_ui_win:min_size(["OK","Cancel"], 70, 30), Ybtn = Yacc + Pad + Hfrm + Pad, - Ok = gs:button(Win, [{x, Pad}, {y, Ybtn}, + OK = gs:button(Win, [{x, Pad}, {y, Ybtn}, {width, Wbtn}, {height, Hbtn}, - {label, {text,"Ok"}}, {font, Font}]), + {label, {text,"OK"}}, {font, Font}]), gs:button(Win, [{x, W-Pad-Wbtn}, {y, Ybtn}, {width, Wbtn}, {height, Hbtn}, {label, {text,"Cancel"}}, {font, Font}]), @@ -175,7 +175,7 @@ create_win(GS, {X, Y}, Type, Mod, Line) -> gs:config(Win, [{width, W}, {height, Hwin}, {map, true}]), #winInfo{type=Type, win=Win, - entries=Entries, trigger=enable, ok=Ok}. + entries=Entries, trigger=enable, ok=OK}. %%-------------------------------------------------------------------- %% update_functions(WinInfo, Funcs) -> WinInfo @@ -229,7 +229,7 @@ handle_event({gs, LB, keypress, window, [Key|_]}, WinInfo) -> Key/='Tab', Key/='Return' -> ignore; true -> - handle_event({gs, LB, click, listbox, ["Ok"]}, WinInfo) + handle_event({gs, LB, click, listbox, ["OK"]}, WinInfo) end; handle_event({gs, Ent, keypress, Data, [Key|_]}, WinInfo) -> case WinInfo#winInfo.type of @@ -249,14 +249,14 @@ handle_event({gs, Ent, keypress, Data, [Key|_]}, WinInfo) -> case next_entry(Ent, WinInfo#winInfo.entries) of last -> gs:config(WinInfo#winInfo.ok, flash), - handle_event({gs, Ent, click, Data, ["Ok"]}, WinInfo); + handle_event({gs, Ent, click, Data, ["OK"]}, WinInfo); Next -> gs:config(Next, {setfocus, true}), ignore end; _Type -> ignore end; -handle_event({gs, _Id, click, _Data, ["Ok"|_]}, WinInfo) -> +handle_event({gs, _Id, click, _Data, ["OK"|_]}, WinInfo) -> case check_input(WinInfo#winInfo.entries) of error -> ignore; Data when WinInfo#winInfo.type/=function -> diff --git a/lib/debugger/src/dbg_ui_edit_win.erl b/lib/debugger/src/dbg_ui_edit_win.erl index badaf4bef4..82f784aa83 100644 --- a/lib/debugger/src/dbg_ui_edit_win.erl +++ b/lib/debugger/src/dbg_ui_edit_win.erl @@ -64,13 +64,13 @@ create_win(GS, {X, Y}, Title, Prompt, {Type, Value}) -> {text, Value}, {keypress, true}]), - %% Ok and Cancel buttons + %% OK and Cancel buttons W = Pad + Wlbl + Went + Pad, {Wbtn, Hbtn} = dbg_ui_win:min_size(["Cancel"], 70, 30), Ybtn = Pad + Hlbl + Pad, Btn = gs:button(Win, [{x, Pad}, {y, Ybtn}, {width, Wbtn}, {height, Hbtn}, - {label, {text,"Ok"}}, {font, Font}]), + {label, {text,"OK"}}, {font, Font}]), gs:button(Win, [{x, W-Pad-Wbtn}, {y, Ybtn}, {width, Wbtn}, {height, Hbtn}, {label, {text,"Cancel"}}, {font, Font}]), @@ -100,8 +100,8 @@ handle_event({gs, _Id, destroy, _Data, _Arg}, _WinInfo) -> stopped; handle_event({gs, Id, keypress, Data, ['Return'|_]}, WinInfo) -> gs:config(WinInfo#winInfo.button, flash), - handle_event({gs, Id, click, Data, ["Ok"]}, WinInfo); -handle_event({gs, _Id, click, _Data, ["Ok"|_]}, WinInfo) -> + handle_event({gs, Id, click, Data, ["OK"]}, WinInfo); +handle_event({gs, _Id, click, _Data, ["OK"|_]}, WinInfo) -> Ent = WinInfo#winInfo.entry, Str = gs:read(Ent, text), Type = WinInfo#winInfo.type, diff --git a/lib/debugger/src/dbg_ui_filedialog_win.erl b/lib/debugger/src/dbg_ui_filedialog_win.erl index 3203991c1f..1eced1104d 100644 --- a/lib/debugger/src/dbg_ui_filedialog_win.erl +++ b/lib/debugger/src/dbg_ui_filedialog_win.erl @@ -100,7 +100,7 @@ create_win(GS, Title, {X,Y}, Mode, Filter, Extra, FileName) -> Opts = [{y, Y4}, {width, Wbtn}, {height, Hbtn}, {font, Font}], case Mode of normal -> - gs:button(Win, [{label, {text,"Ok"}}, {x, Pad}, + gs:button(Win, [{label, {text,"OK"}}, {x, Pad}, {data, select} | Opts]), gs:button(Win, [{label, {text,"Filter"}}, {x, Wlb/2-Wbtn/2}, {data, filter} | Opts]), diff --git a/lib/debugger/src/dbg_ui_interpret.erl b/lib/debugger/src/dbg_ui_interpret.erl index 952e73b537..079eaeb634 100644 --- a/lib/debugger/src/dbg_ui_interpret.erl +++ b/lib/debugger/src/dbg_ui_interpret.erl @@ -145,7 +145,7 @@ interpret_all(State, Dir, [File0|Files]) -> Window = dbg_ui_filedialog_win:get_window(State#state.win), Error = format_error(int:interpretable(File)), Msg = ["Error when interpreting:", File, Error, - "Ok to continue?"], + "OK to continue?"], case tool_utils:confirm(Window, Msg) of ok -> interpret_all(State, Dir, Files); cancel -> true diff --git a/lib/debugger/src/dbg_ui_settings.erl b/lib/debugger/src/dbg_ui_settings.erl index 146aa7e239..38b2ec424f 100644 --- a/lib/debugger/src/dbg_ui_settings.erl +++ b/lib/debugger/src/dbg_ui_settings.erl @@ -136,8 +136,8 @@ default_settings_dir(GS) -> {ok, CWD} = file:get_cwd(), Msg = ["Default directory", DefDir, "does not exist.", - "Click Ok to create it or", - "Cancel to use other directory!"], + "Click OK to create it or", + "Cancel to use other directory."], case tool_utils:confirm(GS, Msg) of ok -> ToolsDir = filename:dirname(DefDir), diff --git a/lib/debugger/src/dbg_wx_break_win.erl b/lib/debugger/src/dbg_wx_break_win.erl index 7ac82c8fb4..062da3937a 100644 --- a/lib/debugger/src/dbg_wx_break_win.erl +++ b/lib/debugger/src/dbg_wx_break_win.erl @@ -82,8 +82,8 @@ create_win(Parent, Pos, function, Mod, _Line) -> wxComboBox:connect(Text, command_text_updated), wxListBox:connect(LB, command_listbox_selected), wxListBox:connect(LB, command_listbox_doubleclicked), - OkId = wxDialog:getAffirmativeId(Win), - OKButt = wxWindow:findWindowById(OkId, [{parent, Win}]), + OKId = wxDialog:getAffirmativeId(Win), + OKButt = wxWindow:findWindowById(OKId, [{parent, Win}]), wxWindow:disable(OKButt), wxDialog:centreOnParent(Win), wxDialog:show(Win), @@ -141,8 +141,8 @@ create_win(Parent, Pos, Type, Mod, Line) -> wxComboBox:setFocus(ModT), wxDialog:connect(Win, command_button_clicked), wxDialog:connect(Win, command_text_updated), - OkId = wxDialog:getAffirmativeId(Win), - OKButt = wxWindow:findWindowById(OkId), + OKId = wxDialog:getAffirmativeId(Win), + OKButt = wxWindow:findWindowById(OKId), wxWindow:disable(OKButt), wxDialog:centreOnParent(Win), wxDialog:show(Win), @@ -180,30 +180,30 @@ handle_event(#wx{id=?wxID_CANCEL}, #winInfo{win=Win}) -> wxDialog:destroy(Win), stopped; handle_event(#wx{event=#wxCommand{type=command_text_updated}}, - #winInfo{type=function, text=Text, ok=Ok}) -> + #winInfo{type=function, text=Text, ok=OK}) -> Module = wxComboBox:getValue(Text), - wxWindow:disable(Ok), + wxWindow:disable(OK), {module, list_to_atom(Module)}; handle_event(#wx{event=#wxCommand{type=command_text_updated}}, - #winInfo{text=Text, ok=Ok, entries=Es}) -> + #winInfo{text=Text, ok=OK, entries=Es}) -> Module = wxComboBox:getValue(Text), case check_input(Es) of - error -> wxWindow:disable(Ok); - _Data when Module =/= "" -> wxWindow:enable(Ok); - _ -> wxWindow:disable(Ok) + error -> wxWindow:disable(OK); + _Data when Module =/= "" -> wxWindow:enable(OK); + _ -> wxWindow:disable(OK) end, ignore; handle_event(#wx{event=#wxCommand{type=command_listbox_selected}}, - #winInfo{type=function, listbox=LB, ok=Ok}) -> + #winInfo{type=function, listbox=LB, ok=OK}) -> case wxListBox:getSelections(LB) of - {N,_} when N > 0 -> wxWindow:enable(Ok); - _ -> wxWindow:disable(Ok) + {N,_} when N > 0 -> wxWindow:enable(OK); + _ -> wxWindow:disable(OK) end, ignore; -handle_event(#wx{id=OKorListBox, event=#wxCommand{type=OkorDoubleClick}}, +handle_event(#wx{id=OKorListBox, event=#wxCommand{type=OKorDoubleClick}}, #winInfo{type=function,win=Win,listbox=LB,funcs=Funcs,text=Text}) when OKorListBox =:= ?wxID_OK; - OkorDoubleClick =:= command_listbox_doubleclicked -> + OKorDoubleClick =:= command_listbox_doubleclicked -> Mod = wxComboBox:getValue(Text), {_, IndexL} = wxListBox:getSelections(LB), Breaks = [[list_to_atom(Mod)|lists:nth(Index+1, Funcs)] || Index <- IndexL], diff --git a/lib/debugger/src/dbg_wx_filedialog_win.erl b/lib/debugger/src/dbg_wx_filedialog_win.erl index 9687efa981..9f45ad0c47 100644 --- a/lib/debugger/src/dbg_wx_filedialog_win.erl +++ b/lib/debugger/src/dbg_wx_filedialog_win.erl @@ -151,7 +151,7 @@ init([Parent, Id, Options0]) -> Bott = wxDialog:createButtonSizer(Dlg, ?wxCANCEL bor ?wxOK), wxDialog:connect(Dlg, command_button_clicked), - %% Ok done + %% OK done Box = wxBoxSizer:new(?wxVERTICAL), wxSizer:add(Box, Top, [{border, 2}, {flag,?wxALL bor ?wxEXPAND}]), wxSizer:add(Box, Dir, [{border, 2}, {flag,?wxALL bor ?wxEXPAND}]), diff --git a/lib/debugger/src/dbg_wx_settings.erl b/lib/debugger/src/dbg_wx_settings.erl index 8f87815949..bc88bdf7da 100644 --- a/lib/debugger/src/dbg_wx_settings.erl +++ b/lib/debugger/src/dbg_wx_settings.erl @@ -83,9 +83,9 @@ default_settings_dir(Win) -> false -> {ok, CWD} = file:get_cwd(), - Msg = ["Default directory", DefDir, "does not exist.", - "Click Ok to create it or", - "Cancel to use other directory!"], + Msg = ["Default directory ", DefDir, " does not exist. ", + "Click OK to create it or ", + "Cancel to use other directory."], case dbg_wx_win:confirm(Win, Msg) of ok -> ToolsDir = filename:dirname(DefDir), diff --git a/lib/debugger/src/dbg_wx_trace_win.erl b/lib/debugger/src/dbg_wx_trace_win.erl index 720b913024..720b913024 100755..100644 --- a/lib/debugger/src/dbg_wx_trace_win.erl +++ b/lib/debugger/src/dbg_wx_trace_win.erl diff --git a/lib/debugger/src/dbg_wx_winman.erl b/lib/debugger/src/dbg_wx_winman.erl index 79dcc47f6f..79dcc47f6f 100755..100644 --- a/lib/debugger/src/dbg_wx_winman.erl +++ b/lib/debugger/src/dbg_wx_winman.erl diff --git a/lib/dialyzer/doc/src/Makefile b/lib/dialyzer/doc/src/Makefile index 45b0ffa5ff..45b0ffa5ff 100755..100644 --- a/lib/dialyzer/doc/src/Makefile +++ b/lib/dialyzer/doc/src/Makefile diff --git a/lib/dialyzer/doc/src/book.xml b/lib/dialyzer/doc/src/book.xml index 0b4e1cb617..0b4e1cb617 100755..100644 --- a/lib/dialyzer/doc/src/book.xml +++ b/lib/dialyzer/doc/src/book.xml diff --git a/lib/dialyzer/doc/src/fascicules.xml b/lib/dialyzer/doc/src/fascicules.xml index 0678195e07..0678195e07 100755..100644 --- a/lib/dialyzer/doc/src/fascicules.xml +++ b/lib/dialyzer/doc/src/fascicules.xml diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 17291b24f7..17291b24f7 100755..100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml diff --git a/lib/dialyzer/doc/src/part.xml b/lib/dialyzer/doc/src/part.xml index 4410840660..4410840660 100755..100644 --- a/lib/dialyzer/doc/src/part.xml +++ b/lib/dialyzer/doc/src/part.xml diff --git a/lib/dialyzer/doc/src/part_notes.xml b/lib/dialyzer/doc/src/part_notes.xml index cb048d55dd..cb048d55dd 100755..100644 --- a/lib/dialyzer/doc/src/part_notes.xml +++ b/lib/dialyzer/doc/src/part_notes.xml diff --git a/lib/dialyzer/doc/src/ref_man.xml b/lib/dialyzer/doc/src/ref_man.xml index ca5410f6b8..ca5410f6b8 100755..100644 --- a/lib/dialyzer/doc/src/ref_man.xml +++ b/lib/dialyzer/doc/src/ref_man.xml diff --git a/lib/dialyzer/info b/lib/dialyzer/info index 9fba4b54ad..9fba4b54ad 100755..100644 --- a/lib/dialyzer/info +++ b/lib/dialyzer/info diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index bcdcf2685d..84b926a17a 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -480,7 +480,7 @@ invalid_contract_warning({M, F, A}, FileLine, SuccType, RecDict) -> extra_range_warning({M, F, A}, FileLine, ExtraRanges, STRange) -> ERangesStr = erl_types:t_to_string(ExtraRanges), STRangeStr = erl_types:t_to_string(STRange), - {?WARN_CONTRACT_TYPES, FileLine, + {?WARN_CONTRACT_SUPERTYPE, FileLine, {extra_range, [M, F, A, ERangesStr, STRangeStr]}}. picky_contract_check(CSig0, Sig0, MFA, FileLine, Contract, RecDict, Acc) -> diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index 30aec59d22..92868b6878 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -2161,13 +2161,21 @@ get_apply_constr(FunLabels, Dst, ArgTypes, #state{callgraph = CG} = State) -> case lists:member(error, MFAs) of true -> error; false -> - Constrs = [begin - State1 = state__new_constraint_context(State), - State2 = get_plt_constr(MFA, Dst, ArgTypes, State1), - state__cs(State2) - end || {ok, MFA} <- MFAs], - ApplyConstr = mk_disj_constraint_list(Constrs), - {ok, state__store_conj(ApplyConstr, State)} + Constrs0 = + [begin + State1 = state__new_constraint_context(State), + try get_plt_constr(MFA, Dst, ArgTypes, State1) of + State2 -> state__cs(State2) + catch + throw:error -> error + end + end || {ok, MFA} <- MFAs], + case [C || C <- Constrs0, C =/= error] of + [] -> throw(error); + Constrs -> + ApplyConstr = mk_disj_constraint_list(Constrs), + {ok, state__store_conj(ApplyConstr, State)} + end end. state__scc(#state{scc = SCC}) -> diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/crash b/lib/dialyzer/test/opaque_SUITE_data/results/crash index 6bdd934169..1ddae5149f 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/crash +++ b/lib/dialyzer/test/opaque_SUITE_data/results/crash @@ -1,5 +1,4 @@ -crash_1.erl:42: The specification for crash_1:empty/0 states that the function might also return crash_1:targetlist() but the inferred return is none() crash_1.erl:45: Record construction #targetlist{list::[]} violates the declared type of field list::'undefined' | crash_1:target() crash_1.erl:48: The call crash_1:get_using_branch2(Branch::maybe_improper_list(),L::'undefined' | crash_1:target()) contains an opaque term as 2nd argument when terms of different types are expected in these positions crash_1.erl:50: The pattern <_Branch, []> can never match the type <maybe_improper_list(),'undefined' | crash_1:target()> diff --git a/lib/dialyzer/test/small_SUITE_data/results/higher_order_discrepancy b/lib/dialyzer/test/small_SUITE_data/results/higher_order_discrepancy new file mode 100644 index 0000000000..7ce440a60d --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/higher_order_discrepancy @@ -0,0 +1,4 @@ + +higher_order_discrepancy.erl:11: The call higher_order_discrepancy:g('foo') will never return since it differs in the 1st argument from the success typing arguments: ('bar') +higher_order_discrepancy.erl:14: Function g/1 has no local return +higher_order_discrepancy.erl:14: The pattern 'bar' can never match the type 'foo' diff --git a/lib/dialyzer/test/small_SUITE_data/results/higher_order_discrepancy_2 b/lib/dialyzer/test/small_SUITE_data/results/higher_order_discrepancy_2 new file mode 100644 index 0000000000..c1c7dbbfcc --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/results/higher_order_discrepancy_2 @@ -0,0 +1,8 @@ + +higher_order_discrepancy_2.erl:11: The call higher_order_discrepancy_2:f('foo') will never return since it differs in the 1st argument from the success typing arguments: ('bar') +higher_order_discrepancy_2.erl:11: The call higher_order_discrepancy_2:g('foo') will never return since it differs in the 1st argument from the success typing arguments: ('baz') +higher_order_discrepancy_2.erl:13: Function f/1 has no local return +higher_order_discrepancy_2.erl:13: The pattern 'bar' can never match the type 'foo' +higher_order_discrepancy_2.erl:14: Function g/1 has no local return +higher_order_discrepancy_2.erl:14: The pattern 'baz' can never match the type 'foo' +higher_order_discrepancy_2.erl:5: Function test/1 has no local return diff --git a/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash b/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash index 191d3d4173..8c9df56a4b 100644 --- a/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash +++ b/lib/dialyzer/test/small_SUITE_data/results/tuple_set_crash @@ -12,4 +12,3 @@ tuple_set_crash.erl:179: The pattern <<AudioVolume:16/integer-little-unit:1,Rest tuple_set_crash.erl:182: The pattern <<Delay:16/integer-little-unit:1,_Padding/binary-unit:8>> can never match the type <<_:8>> tuple_set_crash.erl:62: The pattern {'play_list', _Playlist} can never match the type 'ok' | {'device_properties',[{atom(),_}]} | {'error',[{atom(),_}]} tuple_set_crash.erl:64: The pattern {'error', 17} can never match the type 'ok' | {'device_properties',[{atom(),_}]} | {'error',[{atom(),_}]} -tuple_set_crash.erl:83: The specification for tuple_set_crash:parse_message/1 states that the function might also return {'media_item_url_reply',integer(),binary()} but the inferred return is 'ok' | {'audio_device_info' | 'audio_output_info' | 'audio_target_info' | 'device_properties' | 'error' | 'video_device_info' | 'video_output_info' | 'video_target_info',[{'address' | 'audio_volume' | 'controller_description' | 'controller_name' | 'controller_status' | 'device_id' | 'display_type' | 'fw_version' | 'master_volume' | 'model' | 'output_id' | 'status' | 'target_id',binary() | non_neg_integer()}] | 1..255} diff --git a/lib/dialyzer/test/small_SUITE_data/src/higher_order_discrepancy.erl b/lib/dialyzer/test/small_SUITE_data/src/higher_order_discrepancy.erl new file mode 100644 index 0000000000..ff5ee6bac4 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/higher_order_discrepancy.erl @@ -0,0 +1,14 @@ +-module(higher_order_discrepancy). + +-export([test/1]). + +test(X) -> + F = + case X of + 1 -> fun f/1; + 2 -> fun g/1 + end, + F(foo). + +f(foo) -> ok. +g(bar) -> ok. diff --git a/lib/dialyzer/test/small_SUITE_data/src/higher_order_discrepancy_2.erl b/lib/dialyzer/test/small_SUITE_data/src/higher_order_discrepancy_2.erl new file mode 100644 index 0000000000..4b0d4f6b45 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/higher_order_discrepancy_2.erl @@ -0,0 +1,14 @@ +-module(higher_order_discrepancy_2). + +-export([test/1]). + +test(X) -> + F = + case X of + 1 -> fun f/1; + 2 -> fun g/1 + end, + F(foo). + +f(bar) -> ok. +g(baz) -> ok. diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 43c497f50a..2d8edb1301 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -431,7 +431,7 @@ eval(F) -> </code> <p> -Evaluating an evaluable() <c>E</c> on an argument list <c>A</c> +Applying an evaluable() <c>E</c> to an argument list <c>A</c> is meant in the sense of <c>eval([E|A])</c>.</p> <p> @@ -565,7 +565,7 @@ Pkt = #diameter_packet{} </code> <p> -Reports that the RFC 3539 watchdog state machine has +The RFC 3539 watchdog state machine has transitioned into (<c>up</c>) or out of (<c>down</c>) the open state. If a <c>diameter_packet</c> record is present in an <c>up</c> tuple @@ -576,9 +576,9 @@ connectivity.</p> <p> Note that a single up/down event for a given peer corresponds to -as many peer_up/down callbacks as there are Diameter -applications shared by the peer, as determined during capablilities -exchange. +as many <seealso marker="diameter_app#peer_up">peer_up/peer_down</seealso> +callbacks as there are Diameter applications shared by the peer, +as determined during capablilities exchange. That is, the event communicates connectivity with the peer as a whole while the callbacks communicate connectivity with respect to individual Diameter applications.</p> @@ -597,12 +597,96 @@ transport connection with a peer following <c>reconnect_timer</c> or <c>watchdog_timer</c> expiry.</p> </item> +<tag><c>{closed, Ref, Reason, Config}</c></tag> +<item> +<code> +Ref = transport_ref() +Config = {connect|listen, [transport_opt()]} +</code> + +<p> +Capabilities exchange has failed. <c>Reason</c> can be one of +the following.</p> + +<taglist> + +<tag><c>{'CER', Result, Caps, Pkt}</c></tag> +<item> +<code> +Result = ResultCode | {capabilities_cb, CB, ResultCode|discard} +Caps = #diameter_caps{} +Pkt = #diameter_packet{} +ResultCode = integer() +CB = evaluable() +</code> + +<p> +An incoming CER has been answered with the indicated result code or +discarded. +The capabilities record contains pairs of values for the the local +node and remote peer. +The packet record contains the CER in question. +In the case of rejection by a capabilities callback, the tuple +indicates the rejecting callback.</p> +</item> + +<tag><c>{'CER', Caps, {ResultCode, Pkt}}</c></tag> +<item> +<code> +ResultCode = integer() +Caps = #diameter_caps{} +Pkt = #diameter_packet{} +</code> + +<p> +An incoming CER contained errors and has been answered with the +indicated result code. +The capabilities record contains only values for the the local +node. +The packet record contains the CER in question.</p> +</item> + +<tag><c>{'CEA', Result, Caps, Pkt}</c></tag> +<item> +<code> +Result = integer() | atom() | {capabilities_cb, CB, ResultCode|discard} +Caps = #diameter_caps{} +Pkt = #diameter_packet{} +ResultCode = integer() +</code> + +<p> +An incoming CEA has been rejected for the indicated reason. +An integer-valued <c>Result</c> indicates the result code sent +by the peer. +The capabilities record contains pairs of values for the the local +node and remote peer. +The packet record contains the CEA in question. +In the case of rejection by a capabilities callback, the tuple +indicates the rejecting callback.</p> +</item> + +<tag><c>{'CEA', Caps, Pkt}</c></tag> +<item> +<code> +Caps = #diameter_caps{} +Pkt = #diameter_packet{} +</code> + +<p> +An incoming CER contained errors and has been rejected. +The capabilities record contains only values for the the local node. +The packet record contains the CEA in question.</p> +</item> + +</taglist> +</item> + </taglist> <p> For forward compatibility, a subscriber should be prepared to receive -<c>diameter_event.info</c> of forms other than those documented -above.</p> +info fields of forms other than the above.</p> <marker id="service_name"/> </item> @@ -706,6 +790,31 @@ not over SCTP as implemented by <seealso marker="diameter_sctp">diameter_sctp(3)</seealso>.</p> </item> +<tag><c>{capabilities_cb, evaluable()}</c></tag> +<item> +<p> +A callback invoked upon reception of CER/CEA during capabilities +exchange in order to ask whether or not the connection should +be accepted. +Applied to the transport reference (as returned by <seealso +marker="#add_transport">add_transport/2</seealso>) and +<c>diameter_caps</c> record of the connection. +Returning <c>ok</c> accepts the connection. +Returning <c>integer()</c> causes an incoming +CER to be answered with the specified Result-Code. +Returning <c>discard</c> causes an incoming CER to +be discarded. +Returning <c>unknown</c> is equivalent to returning <c>3010</c>, +DIAMETER_UNKNOWN_PEER. +Returning anything but <c>ok</c> or a 2xxx series result +code causes the transport connection to be broken.</p> + +<p> +Multiple <c>capabilities_cb</c> options can be specified, in which +case the corresponding callbacks are applied until either all return +<c>ok</c> or one does not.</p> +</item> + <tag><c>{watchdog_timer, TwInit}</c></tag> <item> <code> diff --git a/lib/diameter/include/diameter_gen.hrl b/lib/diameter/include/diameter_gen.hrl index d037e1044a..13a6c462af 100644 --- a/lib/diameter/include/diameter_gen.hrl +++ b/lib/diameter/include/diameter_gen.hrl @@ -40,14 +40,14 @@ encode_avps(Name, Rec) -> list_to_binary(encode(Name, Rec)) catch throw: {?MODULE, Reason} -> - diameter_dbg:log({encode, error}, + diameter_lib:log({encode, error}, ?MODULE, ?LINE, {Reason, Name, Rec}), erlang:error(list_to_tuple(Reason ++ [Name])); error: Reason -> Stack = erlang:get_stacktrace(), - diameter_dbg:log({encode, failure}, + diameter_lib:log({encode, failure}, ?MODULE, ?LINE, {Reason, Name, Rec, Stack}), @@ -159,7 +159,7 @@ d_rc(Name, {Avps, {Rec, [] = Failed}}) -> {Rec, Avps, Failed} catch throw: {?MODULE, {AvpName, Reason}} -> - diameter_dbg:log({decode, error}, + diameter_lib:log({decode, error}, ?MODULE, ?LINE, {AvpName, Reason, Rec}), @@ -260,7 +260,7 @@ d(Name, Avp, {Avps, Acc}) -> %% respond sensibly to. Log the occurence for traceability, %% but the peer will also receive info in the resulting %% answer-message. - diameter_dbg:log({decode, failure}, + diameter_lib:log({decode, failure}, ?MODULE, ?LINE, {Reason, Avp, erlang:get_stacktrace()}), diff --git a/lib/diameter/src/Makefile b/lib/diameter/src/Makefile index a057632d15..eea2aa894d 100644 --- a/lib/diameter/src/Makefile +++ b/lib/diameter/src/Makefile @@ -177,7 +177,7 @@ else include $(ERL_TOP)/make/otp_release_targets.mk endif -# Can't $(INSTALL_DIR) more than on directory at a time on Solaris. +# Can't $(INSTALL_DIR) more than one directory at a time on Solaris. release_spec: opt for d in bin ebin examples include src/dict $(TARGET_DIRS:%/=src/%); do \ @@ -209,7 +209,7 @@ gen/diameter_gen_base_accounting.hrl gen/diameter_gen_relay.hrl: \ gen/diameter_gen_base_rfc3588.erl gen/diameter_gen_base_rfc3588.hrl: \ $(COMPILER_MODULES:compiler/%=$(EBIN)/%.$(EMULATOR)) -$(DICT_MODULES:%=$(EBIN)/%.$(EMULATOR)): \ +$(DICT_MODULES:gen/%=$(EBIN)/%.$(EMULATOR)): \ $(INCDIR)/diameter.hrl \ $(INCDIR)/diameter_gen.hrl diff --git a/lib/diameter/src/base/diameter.appup.src b/lib/diameter/src/base/diameter.appup.src index 6d8ceadb92..b1c94d4cc8 100644 --- a/lib/diameter/src/base/diameter.appup.src +++ b/lib/diameter/src/base/diameter.appup.src @@ -20,28 +20,11 @@ {"%VSN%", [ - {"0.9", - [ - {load_module, diameter, soft_purge, soft_purge, []}, - {load_module, diameter_capx, soft_purge, soft_purge, []}, - {load_module, diameter_codec, soft_purge, soft_purge, [diameter_lib]}, - {load_module, diameter_lib, soft_purge, soft_purge, []}, - {load_module, diameter_types, soft_purge, soft_purge, []}, - {load_module, diameter_gen_base_accounting, soft_purge, soft_purge, []}, - {load_module, diameter_gen_base_rfc3588, soft_purge, soft_purge, []}, - {load_module, diameter_gen_relay, soft_purge, soft_purge, []}, - {update, diameter_service, soft, soft_purge, soft_purge, [diameter_lib]}, - {update, diameter_config, soft, soft_purge, soft_purge, []}, - {update, diameter_peer, soft, soft_purge, soft_purge, []}, - {update, diameter_peer_fsm, soft, soft_purge, soft_purge, [diameter_lib]}, - {update, diameter_reg, soft, soft_purge, soft_purge, []}, - {update, diameter_sctp, soft, soft_purge, soft_purge, []}, - {update, diameter_stats, soft, soft_purge, soft_purge, []}, - {update, diameter_sync, soft, soft_purge, soft_purge, []}, - {update, diameter_watchdog, soft, soft_purge, soft_purge, [diameter_lib]} - ] - } + {"0.9", [{restart_application, diameter}]}, + {"0.10", [{restart_application, diameter}]} ], [ + {"0.9", [{restart_application, diameter}]}, + {"0.10", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/src/base/diameter_capx.erl b/lib/diameter/src/base/diameter_capx.erl index 138e76411e..842a9e6103 100644 --- a/lib/diameter/src/base/diameter_capx.erl +++ b/lib/diameter/src/base/diameter_capx.erl @@ -57,9 +57,9 @@ -include("diameter_types.hrl"). -include("diameter_gen_base_rfc3588.hrl"). --define(SUCCESS, ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_SUCCESS'). --define(NOAPP, ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_NO_COMMON_APPLICATION'). --define(NOSECURITY, ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_NO_COMMON_SECURITY'). +-define(SUCCESS, 2001). %% DIAMETER_SUCCESS +-define(NOAPP, 5010). %% DIAMETER_NO_COMMON_APPLICATION +-define(NOSECURITY, 5017). %% DIAMETER_NO_COMMON_SECURITY -define(NO_INBAND_SECURITY, 0). -define(TLS, 1). @@ -96,7 +96,7 @@ try_it([Fun | Args]) -> try apply(Fun, Args) of T -> {ok, T} catch - throw: ?FAILURE(Reason) -> {error, {Reason, Args}} + throw: ?FAILURE(Reason) -> {error, Reason} end. %% mk_caps/2 @@ -224,10 +224,6 @@ rCER(CER, #diameter_service{capabilities = LCaps} = Svc) -> RCaps, CEA#diameter_base_CEA{'Result-Code' = ?SUCCESS})}. -%% TODO: 5.3 of RFC 3588 says we MUST return DIAMETER_NO_COMMON_APPLICATION -%% in the CEA and SHOULD disconnect the transport. However, we have -%% no way to guarantee the send before disconnecting. - build_CEA([], _, _, CEA) -> CEA#diameter_base_CEA{'Result-Code' = ?NOAPP}; @@ -292,25 +288,12 @@ to_cea(CER, Field, CEA) -> %% rCEA/2 -rCEA(#diameter_base_CEA{'Result-Code' = RC} - = CEA, - #diameter_service{capabilities = LCaps} - = Svc) -> - RC == ?SUCCESS orelse ?THROW({'Result-Code', RC}), - +rCEA(CEA, #diameter_service{capabilities = LCaps} = Svc) -> RCaps = capx_to_caps(CEA), SApps = common_applications(LCaps, RCaps, Svc), - - [] == SApps andalso ?THROW(no_common_applications), - IS = common_security(LCaps, RCaps), - [] == IS andalso ?THROW(no_common_security), - - {SApps, IS, RCaps}; - -rCEA(CEA, _Svc) -> - ?THROW({invalid, CEA}). + {SApps, IS, RCaps}. %% capx_to_caps/1 diff --git a/lib/diameter/src/base/diameter_codec.erl b/lib/diameter/src/base/diameter_codec.erl index d88f42fb7c..fe1212b7e0 100644 --- a/lib/diameter/src/base/diameter_codec.erl +++ b/lib/diameter/src/base/diameter_codec.erl @@ -190,26 +190,13 @@ encode_avps(Avps) -> %% msg_header/3 -msg_header(Mod, MsgName, Header) -> - {Code, Flags, ApplId} = h(Mod, MsgName, Header), - {Code, p(Flags, Header), ApplId}. - -%% 6.2 of 3588 requires the same 'P' bit on an answer as on the -%% request. - -p(Flags, #diameter_header{is_request = true, - is_proxiable = P}) -> - Flags band (2#10110000 bor choose(P, 2#01000000, 0)); -p(Flags, _) -> - Flags. - -h(Mod, 'answer-message' = MsgName, Header) -> +msg_header(Mod, 'answer-message' = MsgName, Header) -> ?BASE = Mod, #diameter_header{cmd_code = Code} = Header, {_, Flags, ApplId} = ?BASE:msg_header(MsgName), {Code, Flags, ApplId}; -h(Mod, MsgName, _) -> +msg_header(Mod, MsgName, _) -> Mod:msg_header(MsgName). %% rec2msg/2 @@ -554,8 +541,3 @@ pack_avp(Code, Flags, Vid, Sz, Bin) -> pack_avp(Code, Flags, Sz, Bin) -> Length = Sz + 8, <<Code:32, Flags:8, Length:24, Bin/binary>>. - -%% =========================================================================== - -choose(true, X, _) -> X; -choose(false, _, X) -> X. diff --git a/lib/diameter/src/base/diameter_peer_fsm.erl b/lib/diameter/src/base/diameter_peer_fsm.erl index 282fa2742f..fae5d763dc 100644 --- a/lib/diameter/src/base/diameter_peer_fsm.erl +++ b/lib/diameter/src/base/diameter_peer_fsm.erl @@ -55,7 +55,8 @@ -define(NO_INBAND_SECURITY, 0). -define(TLS, 1). --define(LOOP_TIMEOUT, 2000). +%% A 2xxx series Result-Code. Not necessarily 2001. +-define(IS_SUCCESS(N), 2 == (N) div 1000). %% RFC 3588: %% @@ -142,9 +143,12 @@ init(T) -> proc_lib:init_ack({ok, self()}), gen_server:enter_loop(?MODULE, [], i(T)). -i({WPid, {M, _} = T, Opts, #diameter_service{capabilities = Caps} = Svc0}) -> +i({WPid, T, Opts, #diameter_service{capabilities = Caps} = Svc0}) -> putr(dwa, dwa(Caps)), - {ok, TPid, Svc} = start_transport(T, Opts, Svc0), + {M, Ref} = T, + {[Ts], Rest} = proplists:split(Opts, [capabilities_cb]), + putr(capabilities_cb, {Ref, [F || {_,F} <- Ts]}), + {ok, TPid, Svc} = start_transport(T, Rest, Svc0), erlang:monitor(process, TPid), erlang:monitor(process, WPid), #state{parent = WPid, @@ -198,10 +202,13 @@ handle_info(T, #state{} = State) -> ?LOG(stop, T), x(T, State) catch - throw: {?MODULE, Tag, Reason} -> + {?MODULE, Tag, Reason} -> ?LOG(Tag, {Reason, T}), {stop, {shutdown, Reason}, State} end. +%% The form of the exception caught here is historical. It's +%% significant that it's not a 2-tuple, as in ?FAILURE(Reason), +%% since these are caught elsewhere. x(Reason, #state{} = S) -> close_wd(Reason, S), @@ -226,6 +233,9 @@ putr(Key, Val) -> getr(Key) -> get({?MODULE, Key}). +eraser(Key) -> + erase({?MODULE, Key}). + %% transition/2 %% Connection to peer. @@ -316,9 +326,10 @@ send_CER(#state{mode = {connect, Remote}, service = #diameter_service{capabilities = Caps}, transport = TPid} = S) -> - req_send_CER(Caps#diameter_caps.origin_host, Remote) + OH = Caps#diameter_caps.origin_host, + req_send_CER(OH, Remote) orelse - close(connected, S), + close({already_connected, Remote, Caps}, S), CER = build_CER(S), ?LOG(send, 'CER'), send(TPid, encode(CER)), @@ -460,20 +471,20 @@ handle_request(Type, #diameter_packet{} = Pkt, S) -> %% send_answer/3 send_answer(Type, ReqPkt, #state{transport = TPid} = S) -> - #diameter_packet{header = #diameter_header{version = V, - end_to_end_id = Eid, - hop_by_hop_id = Hid, - is_proxiable = P}, + #diameter_packet{header = H, transport_data = TD} = ReqPkt, - {Answer, PostF} = build_answer(Type, V, ReqPkt, S), + {Msg, PostF} = build_answer(Type, ReqPkt, S), - Pkt = #diameter_packet{header = #diameter_header{version = V, - end_to_end_id = Eid, - hop_by_hop_id = Hid, - is_proxiable = P}, - msg = Answer, + %% An answer message clears the R and T flags and retains the P + %% flag. The E flag is set at encode. + Pkt = #diameter_packet{header + = H#diameter_header{version = ?DIAMETER_VERSION, + is_request = false, + is_error = undefined, + is_retransmitted = false}, + msg = Msg, transport_data = TD}, send(TPid, diameter_codec:encode(?BASE, Pkt)), @@ -484,56 +495,104 @@ eval([F|A], S) -> eval(ok, S) -> S. -%% build_answer/4 +%% build_answer/3 build_answer('CER', - ?DIAMETER_VERSION, #diameter_packet{msg = CER, - header = #diameter_header{is_error = false}, + header = #diameter_header{version + = ?DIAMETER_VERSION, + is_error = false}, errors = []} = Pkt, - #state{service = Svc} - = S) -> - #diameter_service{capabilities = #diameter_caps{origin_host = OH}} - = Svc, - - {SupportedApps, - #diameter_caps{origin_host = DH} = RCaps, - #diameter_base_CEA{'Result-Code' = RC} - = CEA} + S) -> + {SupportedApps, RCaps, #diameter_base_CEA{'Result-Code' = RC, + 'Inband-Security-Id' = IS} + = CEA} = recv_CER(CER, S), + #diameter_caps{origin_host = {OH, DH}} + = Caps + = capz(caps(S), RCaps), + try 2001 == RC %% DIAMETER_SUCCESS - orelse ?THROW({sent_CEA, RC}), + orelse ?THROW(RC), register_everywhere({?MODULE, connection, OH, DH}) - orelse ?THROW({election_lost, 4003}), - #diameter_base_CEA{'Inband-Security-Id' = [IS]} - = CEA, - {CEA, [fun open/5, Pkt, SupportedApps, RCaps, {accept, IS}]} + orelse ?THROW(4003), %% DIAMETER_ELECTION_LOST + caps_cb(Caps) + of + N -> {cea(CEA, N), [fun open/5, Pkt, + SupportedApps, + Caps, + {accept, hd([_] = IS)}]} catch - ?FAILURE({Reason, RC}) -> - {answer('CER', S) ++ [{'Result-Code', RC}], - [fun close/2, {'CER', Reason, DH}]} + ?FAILURE(Reason) -> + rejected(Reason, {'CER', Reason, Caps, Pkt}, S) end; %% The error checks below are similar to those in diameter_service for %% other messages. Should factor out the commonality. -build_answer(Type, V, #diameter_packet{header = H, errors = Es} = Pkt, S) -> - FailedAvp = failed_avp([A || {_,A} <- Es]), - Ans = answer(answer(Type, S), V, H, Es), - {set(Ans, FailedAvp), if 'CER' == Type -> - [fun close/2, {Type, V, Pkt}]; - true -> - ok - end}. +build_answer(Type, + #diameter_packet{header = H, + errors = Es} + = Pkt, + S) -> + RC = rc(H, Es), + {answer(Type, RC, Es, S), post(Type, RC, Pkt, S)}. + +cea(CEA, ok) -> + CEA; +cea(CEA, 2001) -> + CEA; +cea(CEA, RC) -> + CEA#diameter_base_CEA{'Result-Code' = RC}. + +post('CER' = T, RC, Pkt, S) -> + [fun close/2, {T, caps(S), {RC, Pkt}}]; +post(_, _, _, _) -> + ok. + +rejected({capabilities_cb, _F, Reason}, T, S) -> + rejected(Reason, T, S); + +rejected(discard, T, S) -> + close(T, S); +rejected({N, Es}, T, S) -> + {answer('CER', N, Es, S), [fun close/2, T]}; +rejected(N, T, S) -> + rejected({N, []}, T, S). + +answer(Type, RC, Es, S) -> + set(answer(Type, RC, S), failed_avp([A || {_,A} <- Es])). + +answer(Type, RC, S) -> + answer_message(answer(Type, S), RC). +%% answer_message/2 + +answer_message([_ | Avps], RC) + when 3000 =< RC, RC < 4000 -> + ['answer-message', {'Result-Code', RC} + | lists:filter(fun is_origin/1, Avps)]; + +answer_message(Msg, RC) -> + Msg ++ [{'Result-Code', RC}]. + +is_origin({N, _}) -> + N == 'Origin-Host' + orelse N == 'Origin-Realm' + orelse N == 'Origin-State-Id'. + +%% failed_avp/1 + failed_avp([] = No) -> No; failed_avp(Avps) -> [{'Failed-AVP', [[{'AVP', Avps}]]}]. +%% set/2 + set(Ans, []) -> Ans; set(['answer-message' | _] = Ans, FailedAvp) -> @@ -541,18 +600,22 @@ set(['answer-message' | _] = Ans, FailedAvp) -> set([_|_] = Ans, FailedAvp) -> Ans ++ FailedAvp. -answer([_, OH, OR | _], _, #diameter_header{is_error = true}, _) -> - ['answer-message', OH, OR, {'Result-Code', 3008}]; +%% rc/2 + +rc(#diameter_header{is_error = true}, _) -> + 3008; %% DIAMETER_INVALID_HDR_BITS -answer([_, OH, OR | _], _, _, [Bs|_]) +rc(_, [Bs|_]) when is_bitstring(Bs) -> - ['answer-message', OH, OR, {'Result-Code', 3009}]; + 3009; %% DIAMETER_INVALID_HDR_BITS -answer(Ans, ?DIAMETER_VERSION, _, Es) -> - Ans ++ [{'Result-Code', rc(Es)}]; +rc(#diameter_header{version = ?DIAMETER_VERSION}, Es) -> + rc(Es); -answer(Ans, _, _, _) -> - Ans ++ [{'Result-Code', 5011}]. %% DIAMETER_UNSUPPORTED_VERSION +rc(_, _) -> + 5011. %% DIAMETER_UNSUPPORTED_VERSION + +%% rc/1 rc([]) -> 2001; %% DIAMETER_SUCCESS @@ -595,12 +658,14 @@ a('CER', #diameter_caps{vendor_id = Vid, origin_host = Host, origin_realm = Realm, host_ip_address = Addrs, - product_name = Name}) -> + product_name = Name, + origin_state_id = OSI}) -> ['CEA', {'Origin-Host', Host}, {'Origin-Realm', Realm}, {'Host-IP-Address', Addrs}, {'Vendor-Id', Vid}, - {'Product-Name', Name}]; + {'Product-Name', Name}, + {'Origin-State-Id', OSI}]; a('DPR', #diameter_caps{origin_host = Host, origin_realm = Realm}) -> @@ -615,27 +680,25 @@ recv_CER(CER, #state{service = Svc}) -> %% handle_CEA/1 -handle_CEA(#diameter_packet{header = #diameter_header{version = V}, - bin = Bin} +handle_CEA(#diameter_packet{bin = Bin} = Pkt, #state{service = #diameter_service{capabilities = LCaps}} = S) when is_binary(Bin) -> ?LOG(recv, 'CEA'), - ?DIAMETER_VERSION == V orelse close({version, V}, S), - - #diameter_packet{msg = CEA, errors = Errors} + #diameter_packet{msg = CEA} = DPkt = diameter_codec:decode(?BASE, Pkt), - [] == Errors orelse close({errors, Errors}, S), + {SApps, IS, RCaps} = recv_CEA(DPkt, S), - {SApps, [IS], #diameter_caps{origin_host = DH} = RCaps} - = recv_CEA(CEA, S), + #diameter_caps{origin_host = {OH, DH}} + = Caps + = capz(LCaps, RCaps), - #diameter_caps{origin_host = OH} - = LCaps, + #diameter_base_CEA{'Result-Code' = RC} + = CEA, %% Ensure that we don't already have a connection to the peer in %% question. This isn't the peer election of 3588 except in the @@ -643,60 +706,101 @@ handle_CEA(#diameter_packet{header = #diameter_header{version = V}, %% receive a CER/CEA, the first that arrives wins the right to a %% connection with the peer. - register_everywhere({?MODULE, connection, OH, DH}) - orelse close({'CEA', DH}, S), - - open(DPkt, SApps, RCaps, {connect, IS}, S). + try + ?IS_SUCCESS(RC) + orelse ?THROW(RC), + [] == SApps + andalso ?THROW(no_common_application), + [] == IS + andalso ?THROW(no_common_security), + register_everywhere({?MODULE, connection, OH, DH}) + orelse ?THROW(election_lost), + caps_cb(Caps) + of + _ -> open(DPkt, SApps, Caps, {connect, hd([_] = IS)}, S) + catch + ?FAILURE(Reason) -> close({'CEA', Reason, Caps, DPkt}, S) + end. +%% Check more than the result code since the peer could send success +%% regardless. If not 2001 then a peer_up callback could do anything +%% required. It's not unimaginable that a peer agreeing to TLS after +%% capabilities exchange could send DIAMETER_LIMITED_SUCCESS = 2002, +%% even if this isn't required by RFC 3588. %% recv_CEA/2 -recv_CEA(CEA, #state{service = Svc} = S) -> - case diameter_capx:recv_CEA(CEA, Svc) of - {ok, {_,_}} -> %% return from old code - close({'CEA', update}, S); - {ok, {[], _, _}} -> - close({'CEA', no_common_application}, S); - {ok, {_, [], _}} -> - close({'CEA', no_common_security}, S); - {ok, {_,_,_} = T} -> - T; - {error, Reason} -> - close({'CEA', Reason}, S) +recv_CEA(#diameter_packet{header = #diameter_header{version + = ?DIAMETER_VERSION, + is_error = false}, + msg = CEA, + errors = []}, + #state{service = Svc}) -> + {ok, T} = diameter_capx:recv_CEA(CEA, Svc), + T; + +recv_CEA(Pkt, S) -> + close({'CEA', caps(S), Pkt}, S). + +caps(#diameter_service{capabilities = Caps}) -> + Caps; +caps(#state{service = Svc}) -> + caps(Svc). + +%% caps_cb/1 + +caps_cb(Caps) -> + {Ref, Ts} = eraser(capabilities_cb), + ccb(Ts, [Ref, Caps]). + +ccb([], _) -> + ok; +ccb([F | Rest], T) -> + case diameter_lib:eval([F|T]) of + ok -> + ccb(Rest, T); + N when ?IS_SUCCESS(N) -> %% 2xxx result code: accept immediately + N; + Res -> + ?THROW({capabilities_cb, F, rejected(Res)}) end. +%% Note that returning 2xxx causes the capabilities exchange to be +%% accepted directly, without further callbacks. + +rejected(discard = T) -> + T; +rejected(unknown) -> + 3010; %% DIAMETER_UNKNOWN_PEER +rejected(N) + when is_integer(N) -> + N. %% open/5 -open(Pkt, SupportedApps, RCaps, {Type, IS}, #state{parent = Pid, - service = Svc} - = S) -> - #diameter_service{capabilities = #diameter_caps{origin_host = OH, - inband_security_id = LS} - = LCaps} - = Svc, - #diameter_caps{origin_host = DH} - = RCaps, +open(Pkt, SupportedApps, Caps, {Type, IS}, #state{parent = Pid} = S) -> + #diameter_caps{origin_host = {_,_} = H, + inband_security_id = {LS,_}} + = Caps, - tls_ack(lists:member(?TLS, LS), Type, IS, S), - Pid ! {open, self(), {OH,DH}, {capz(LCaps, RCaps), SupportedApps, Pkt}}, + tls_ack(lists:member(?TLS, LS), Caps, Type, IS, S), + Pid ! {open, self(), H, {Caps, SupportedApps, Pkt}}, S#state{state = 'Open'}. %% We've advertised TLS support: tell the transport the result %% and expect a reply when the handshake is complete. -tls_ack(true, Type, IS, #state{transport = TPid} = S) -> +tls_ack(true, Caps, Type, IS, #state{transport = TPid} = S) -> Ref = make_ref(), - MRef = erlang:monitor(process, TPid), TPid ! {diameter, {tls, Ref, Type, IS == ?TLS}}, receive {diameter, {tls, Ref}} -> - erlang:demonitor(MRef, [flush]); - {'DOWN', MRef, process, _, _} = T -> - close({tls_ack, T}, S) + ok; + {'DOWN', _, process, TPid, Reason} -> + close({tls_ack, Reason, Caps}, S) end; %% Or not. Don't send anything to the transport so that transports %% not supporting TLS work as before without modification. -tls_ack(false, _, _, _) -> +tls_ack(false, _, _, _, _) -> ok. capz(#diameter_caps{} = L, #diameter_caps{} = R) -> diff --git a/lib/diameter/src/base/diameter_service.erl b/lib/diameter/src/base/diameter_service.erl index 421e36ccf5..7adcf1c265 100644 --- a/lib/diameter/src/base/diameter_service.erl +++ b/lib/diameter/src/base/diameter_service.erl @@ -983,7 +983,8 @@ peer_cb(MFA, Alias) -> connection_down(Pid, #state{peerT = PeerT, connT = ConnT} = S) -> - #peer{conn = TPid} + #peer{op_state = ?STATE_UP, %% assert + conn = TPid} = P = fetch(PeerT, Pid), @@ -993,6 +994,9 @@ connection_down(Pid, #state{peerT = PeerT, %% connection_down/3 +connection_down(#peer{op_state = ?STATE_DOWN}, _, S) -> + S; + connection_down(#peer{conn = TPid, op_state = ?STATE_UP} = P, @@ -1034,13 +1038,23 @@ down_conn(Id, Alias, TC, {SvcName, Apps}) -> %% Peer process has died. -peer_down(Pid, _Reason, #state{peerT = PeerT} = S) -> +peer_down(Pid, Reason, #state{peerT = PeerT} = S) -> P = fetch(PeerT, Pid), ets:delete_object(PeerT, P), + closed(Reason, P, S), restart(P,S), peer_down(P,S). -%% peer_down/2 +%% Send an event at connection establishment failure. +closed({shutdown, {close, _TPid, Reason}}, + #peer{op_state = ?STATE_DOWN, + ref = Ref, + type = Type, + options = Opts}, + #state{service_name = SvcName}) -> + send_event(SvcName, {closed, Ref, Reason, {type(Type), Opts}}); +closed(_, _, _) -> + ok. %% The peer has never come up ... peer_down(#peer{conn = B}, S) @@ -1048,27 +1062,9 @@ peer_down(#peer{conn = B}, S) S; %% ... or it has. -peer_down(#peer{ref = Ref, - conn = TPid, - type = Type, - options = Opts} - = P, - #state{service_name = SvcName, - connT = ConnT} - = S) -> - #conn{caps = Caps} - = C - = fetch(ConnT, TPid), +peer_down(#peer{conn = TPid} = P, #state{connT = ConnT} = S) -> + #conn{} = C = fetch(ConnT, TPid), ets:delete_object(ConnT, C), - try - pd(P,C,S) - after - send_event(SvcName, {closed, Ref, {TPid, Caps}, {type(Type), Opts}}) - end. - -pd(#peer{op_state = ?STATE_DOWN}, _, S) -> - S; -pd(#peer{op_state = ?STATE_UP} = P, C, S) -> connection_down(P,C,S). %% restart/2 @@ -1259,11 +1255,11 @@ send_request({TPid, Caps, App}, Msg, Opts, Caller, SvcName) -> #diameter_app{module = ModX} = App, - Pkt = make_packet(Msg), + Pkt = make_request_packet(Msg), case cb(ModX, prepare_request, [Pkt, SvcName, {TPid, Caps}]) of {send, P} -> - send_request(make_packet(P, Pkt), + send_request(make_request_packet(P, Pkt), TPid, Caps, App, @@ -1278,70 +1274,73 @@ send_request({TPid, Caps, App}, Msg, Opts, Caller, SvcName) -> ?ERROR({invalid_return, prepare_request, App, T}) end. -%% make_packet/1 +%% make_request_packet/1 %% %% Turn an outgoing request as passed to call/4 into a diameter_packet %% record in preparation for a prepare_request callback. -make_packet(Bin) +make_request_packet(Bin) when is_binary(Bin) -> #diameter_packet{header = diameter_codec:decode_header(Bin), bin = Bin}; -make_packet(#diameter_packet{msg = [#diameter_header{} = Hdr | Avps]} = Pkt) -> - Pkt#diameter_packet{msg = [make_header(Hdr) | Avps]}; +make_request_packet(#diameter_packet{msg = [#diameter_header{} = Hdr | Avps]} + = Pkt) -> + Pkt#diameter_packet{msg = [make_request_header(Hdr) | Avps]}; -make_packet(#diameter_packet{header = Hdr} = Pkt) -> - Pkt#diameter_packet{header = make_header(Hdr)}; +make_request_packet(#diameter_packet{header = Hdr} = Pkt) -> + Pkt#diameter_packet{header = make_request_header(Hdr)}; -make_packet(Msg) -> - make_packet(#diameter_packet{msg = Msg}). +make_request_packet(Msg) -> + make_request_packet(#diameter_packet{msg = Msg}). -%% make_header/1 +%% make_request_header/1 -make_header(undefined) -> +make_request_header(undefined) -> Seq = diameter_session:sequence(), - make_header(#diameter_header{end_to_end_id = Seq, - hop_by_hop_id = Seq}); + make_request_header(#diameter_header{end_to_end_id = Seq, + hop_by_hop_id = Seq}); -make_header(#diameter_header{version = undefined} = Hdr) -> - make_header(Hdr#diameter_header{version = ?DIAMETER_VERSION}); +make_request_header(#diameter_header{version = undefined} = Hdr) -> + make_request_header(Hdr#diameter_header{version = ?DIAMETER_VERSION}); -make_header(#diameter_header{end_to_end_id = undefined} = H) -> +make_request_header(#diameter_header{end_to_end_id = undefined} = H) -> Seq = diameter_session:sequence(), - make_header(H#diameter_header{end_to_end_id = Seq}); + make_request_header(H#diameter_header{end_to_end_id = Seq}); -make_header(#diameter_header{hop_by_hop_id = undefined} = H) -> +make_request_header(#diameter_header{hop_by_hop_id = undefined} = H) -> Seq = diameter_session:sequence(), - make_header(H#diameter_header{hop_by_hop_id = Seq}); + make_request_header(H#diameter_header{hop_by_hop_id = Seq}); -make_header(#diameter_header{} = Hdr) -> +make_request_header(#diameter_header{} = Hdr) -> Hdr; -make_header(T) -> +make_request_header(T) -> ?ERROR({invalid_header, T}). -%% make_packet/2 +%% make_request_packet/2 %% %% Reconstruct a diameter_packet from the return value of %% prepare_request or prepare_retransmit callback. -make_packet(Bin, _) +make_request_packet(Bin, _) when is_binary(Bin) -> - make_packet(Bin); + make_request_packet(Bin); -make_packet(#diameter_packet{msg = [#diameter_header{} | _]} = Pkt, _) -> +make_request_packet(#diameter_packet{msg = [#diameter_header{} | _]} + = Pkt, + _) -> Pkt; %% Returning a diameter_packet with no header from a prepare_request %% or prepare_retransmit callback retains the header passed into it. %% This is primarily so that the end to end and hop by hop identifiers %% are retained. -make_packet(#diameter_packet{header = Hdr} = Pkt, +make_request_packet(#diameter_packet{header = Hdr} = Pkt, #diameter_packet{header = Hdr0}) -> Pkt#diameter_packet{header = fold_record(Hdr0, Hdr)}; -make_packet(Msg, Pkt) -> +make_request_packet(Msg, Pkt) -> Pkt#diameter_packet{msg = Msg}. %% fold_record/2 @@ -1533,7 +1532,7 @@ retransmit({TPid, Caps, #diameter_app{alias = Alias} = App}, case cb(App, prepare_retransmit, [Pkt, SvcName, {TPid, Caps}]) of {send, P} -> - retransmit(make_packet(P, Pkt), TPid, Caps, Req, Timeout); + retransmit(make_request_packet(P, Pkt), TPid, Caps, Req, Timeout); {discard, Reason} -> ?THROW(Reason); discard -> @@ -1946,7 +1945,7 @@ reply(Msg, Dict, TPid, #diameter_packet{errors = Es, = ReqPkt) when [] == Es; is_record(hd(Msg), diameter_header) -> - Pkt = diameter_codec:encode(Dict, make_reply_packet(Msg, ReqPkt)), + Pkt = diameter_codec:encode(Dict, make_answer_packet(Msg, ReqPkt)), incr(send, Pkt, Dict, TPid), %% count result codes in sent answers send(TPid, Pkt#diameter_packet{transport_data = TD}); @@ -1957,18 +1956,19 @@ reply(Msg, Dict, TPid, #diameter_packet{errors = [H|_] = Es} = Pkt) -> TPid, Pkt#diameter_packet{errors = []}). -%% make_reply_packet/2 +%% make_answer_packet/2 %% Binaries and header/avp lists are sent as-is. -make_reply_packet(Bin, _) +make_answer_packet(Bin, _) when is_binary(Bin) -> #diameter_packet{bin = Bin}; -make_reply_packet([#diameter_header{} | _] = Msg, _) -> +make_answer_packet([#diameter_header{} | _] = Msg, _) -> #diameter_packet{msg = Msg}; %% Otherwise a reply message clears the R and T flags and retains the -%% P flag. The E flag will be set at encode. -make_reply_packet(Msg, #diameter_packet{header = ReqHdr}) -> +%% P flag. The E flag will be set at encode. 6.2 of 3588 requires the +%% same P flag on an answer as on the request. +make_answer_packet(Msg, #diameter_packet{header = ReqHdr}) -> Hdr = ReqHdr#diameter_header{version = ?DIAMETER_VERSION, is_request = false, is_error = undefined, diff --git a/lib/diameter/src/base/diameter_watchdog.erl b/lib/diameter/src/base/diameter_watchdog.erl index b7c1491f4b..6dc53d9f31 100644 --- a/lib/diameter/src/base/diameter_watchdog.erl +++ b/lib/diameter/src/base/diameter_watchdog.erl @@ -179,11 +179,11 @@ transition({close, TPid, _Reason}, #watchdog{transport = TPid}) -> %% state okay as the result of the Peer State Machine reaching the %% Open state. %% -%% If we're an acceptor then we may be resuming a connection that went -%% down in another acceptor process, in which case this is the -%% transition below, from down into reopen. That is, it's not until -%% we know the identity of the peer (ie. now) that we know that we're -%% in state down rather than initial. +%% If we're accepting then we may be resuming a connection that went +%% down in another watchdog process, in which case this is the +%% transition below, from down to reopen. That is, it's not until we +%% know the identity of the peer (ie. now) that we know that we're in +%% state down rather than initial. transition({open, TPid, Hosts, T} = Open, #watchdog{transport = TPid, diff --git a/lib/diameter/test/Makefile b/lib/diameter/test/Makefile index 69bcabbfbb..97d9069f4a 100644 --- a/lib/diameter/test/Makefile +++ b/lib/diameter/test/Makefile @@ -58,16 +58,18 @@ SUITES = $(SUITE_MODULES:diameter_%_SUITE=%) # target like 'all' below. Target release_tests only installs source. ERL_COMPILE_FLAGS += +warn_export_vars \ +warn_unused_vars \ - -DDIAMETER_CT=true \ + -I ../include \ -I ../src/gen # ---------------------------------------------------- # Targets # ---------------------------------------------------- -all: $(SUITES) +all: opt -beam tests debug opt: $(TARGET_FILES) +run: $(SUITES) + +debug opt: $(TARGET_FILES) clean: rm -f $(TARGET_FILES) @@ -94,22 +96,22 @@ help: @echo "Useful targets:" @echo @echo " all:" + @echo " Compile all test suites." + @echo + @echo " run:" @echo " Compile and run all test suites." @echo @echo " $(SUITES):" @echo " Compile and run a specific test suite." @echo - @echo " beam:" - @echo " Compile all test-code." - @echo @echo " clean | realclean:" @echo " Remove generated files." @echo @echo " info:" - @echo " Echo some interesting variables." + @echo " Echo some relevant variables." @echo ======================================== -.PHONY: all beam clean debug docs help info opt realclean tests +.PHONY: all run clean debug docs help info opt realclean # ---------------------------------------------------- # Special Targets @@ -118,12 +120,12 @@ help: # Exit with a non-zero status if the output looks to indicate failure. # diameter_ct:run/1 itself can't tell (it seems). The absolute -pa is # because ct will change directories. -$(SUITES): log tests +$(SUITES): log opt $(ERL) -noshell \ - -pa $(realpath ../ebin) \ - -sname diameter_test_$@ \ - -s diameter_ct run diameter_$@_SUITE \ - -s init stop \ + -pa $(realpath ../ebin) \ + -sname diameter_test_$@ \ + -s diameter_ct run diameter_$@_SUITE \ + -s init stop \ | awk '1{rc=0} {print} / FAILED /{rc=1} END{exit rc}' # Shorter in sed but requires a GNU extension (ie. Q). @@ -136,6 +138,9 @@ log: # Release Targets # ---------------------------------------------------- +/%: % force + sed -f release.sed $< > $(RELSYSDIR)$@ + ifeq ($(ERL_TOP),) include $(DIAMETER_TOP)/make/release_targets.mk else @@ -151,10 +156,16 @@ release_tests_spec: $(INSTALL_DATA) $(TEST_SPEC_FILE) \ $(COVER_SPEC_FILE) \ $(HRL_FILES) \ - $(ERL_FILES) \ $(RELSYSDIR) + $(MAKE) $(ERL_FILES:%=/%) + +force: .PHONY: release_spec release_docs_spec release_test_specs +.PHONY: force + +# Can't just make $(ERL_FILES:%=/%) phony since then implicit rule +# searching is skipped. # ---------------------------------------------------- @@ -163,7 +174,7 @@ depend: depend.mk # Generate dependencies makefile. depend.mk: depend.sed $(MODULES:%=%.erl) Makefile (for f in $(MODULES); do \ - sed -f $< $$f.erl | sed "s@/@/$$f@"; \ + (echo $$f; cat $$f.erl) | sed -f $<; \ done) \ > $@ diff --git a/lib/diameter/test/depend.sed b/lib/diameter/test/depend.sed index a399eb45f0..95dca44984 100644 --- a/lib/diameter/test/depend.sed +++ b/lib/diameter/test/depend.sed @@ -18,14 +18,24 @@ # # -# Extract local include dependencies from .erl files. The output is massaged -# further in Makefile. +# Extract local include dependencies from an .erl file. The first +# input line is the module name. # -/^-include/!d +# Store the module name in the hold space. +1{ + h + d +} + +# Throw away everything but local includes. /^-include_lib/d +/^-include/!d /diameter_gen/d +/diameter\./d +# Output a dependency of the beam on the included file. s@^-include("@@ s@".*@@ -s@^@$(EBIN)/.$(EMULATOR): @ +G +s@^\(.*\)\n\(.*\)@$(EBIN)/\2.$(EMULATOR): \1@ diff --git a/lib/diameter/test/diameter_app_SUITE.erl b/lib/diameter/test/diameter_app_SUITE.erl index d5ecdea291..7f53a4ddd4 100644 --- a/lib/diameter/test/diameter_app_SUITE.erl +++ b/lib/diameter/test/diameter_app_SUITE.erl @@ -134,13 +134,16 @@ release(Config) -> Rel = {release, {"diameter test release", fetch(vsn, App)}, {erts, erlang:system_info(version)}, - [{A, appvsn(A)} || A <- fetch(applications, App)]}, + [{A, appvsn(A)} || A <- [sasl | fetch(applications, App)]]}, Dir = fetch(priv_dir, Config), ok = write_file(filename:join([Dir, "diameter_test.rel"]), Rel), {ok, _, []} = systools:make_script("diameter_test", [{path, [Dir]}, {outdir, Dir}, silent]). +%% sasl need to be included to avoid a missing_sasl warning, error +%% in the case of relup/1. + appvsn(Name) -> [{application, Name, App}] = diameter_util:consult(Name, app), fetch(vsn, App). @@ -208,7 +211,7 @@ relup(Config) -> App = fetch(app, Config), Rel = [{erts, erlang:system_info(version)} - | [{A, appvsn(A)} || A <- fetch(applications, App)]], + | [{A, appvsn(A)} || A <- [sasl | fetch(applications, App)]]], Dir = fetch(priv_dir, Config), @@ -216,12 +219,15 @@ relup(Config) -> UpFrom = acc_rel(Dir, Rel, Up), DownTo = acc_rel(Dir, Rel, Down), - {[Name], [Name], UpFrom, DownTo} %% no intersections + {[Name], [Name], [], []} %% no current in up/down and go both ways = {[Name] -- UpFrom, [Name] -- DownTo, UpFrom -- DownTo, DownTo -- UpFrom}, + [[], []] = [S -- sets:to_list(sets:from_list(S)) + || S <- [UpFrom, DownTo]], + {ok, _, _, []} = systools:make_relup(Name, UpFrom, DownTo, [{path, [Dir]}, {outdir, Dir}, silent]). diff --git a/lib/diameter/test/diameter_capx_SUITE.erl b/lib/diameter/test/diameter_capx_SUITE.erl new file mode 100644 index 0000000000..e6b1558bf6 --- /dev/null +++ b/lib/diameter/test/diameter_capx_SUITE.erl @@ -0,0 +1,432 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Tests of capabilities exchange between Diameter nodes. In +%% particular, of error and event handling. +%% + +-module(diameter_capx_SUITE). + +-export([suite/0, + all/0, + groups/0, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2]). + +%% testcases +-export([start/1, + start_services/1, + add_listeners/1, + s_no_common_application/1, + c_no_common_application/1, + s_no_common_security/1, + c_no_common_security/1, + s_unknown_peer/1, + c_unknown_peer/1, + s_unable/1, + c_unable/1, + s_client_reject/1, + c_client_reject/1, + remove_listeners/1, + stop_services/1, + stop/1]). + +%% diameter callbacks +-export([peer_up/4, + peer_down/4]). + +-include("diameter.hrl"). +-include("diameter_gen_base_rfc3588.hrl"). + +%% =========================================================================== + +-define(util, diameter_util). + +-define(CLIENT, client). +-define(SERVER, server). + +-define(ADDR, {127,0,0,1}). + +-define(REALM, "erlang.org"). +-define(HOST(Name), Name ++ "." ++ ?REALM). + +%% Config for diameter:start_service/2. +-define(SERVICE(Name), + [{'Origin-Realm', ?REALM}, + {'Host-IP-Address', [?ADDR]}, + {'Vendor-Id', 12345}, + {'Product-Name', "OTP/diameter"}, + {'Auth-Application-Id', [?DIAMETER_APP_ID_COMMON]}, + {'Acct-Application-Id', [?DIAMETER_APP_ID_ACCOUNTING]} + | [{application, [{alias, A}, + {dictionary, D}, + {module, [?MODULE, A]}]} + || {A,D} <- [{common, ?DIAMETER_DICT_COMMON}, + {accounting, ?DIAMETER_DICT_ACCOUNTING}]]]). + +-define(A, list_to_atom). +-define(L, atom_to_list). + +-define(event, #diameter_event). +-define(caps, #diameter_caps). +-define(packet, #diameter_packet). + +-define(cea, #diameter_base_CEA). +-define(answer_message, #'diameter_base_answer-message'). + +%% =========================================================================== + +suite() -> + [{timetrap, {seconds, 10}}]. + +all() -> + [start, start_services, add_listeners + | [{group, N} || {N, _, _} <- groups()]] + ++ [remove_listeners, stop_services, stop]. + +groups() -> + Ts = testcases(), + [{grp(P), P, Ts} || P <- [[], [parallel]]]. + +grp([]) -> + sequential; +grp([parallel = P]) -> + P. + +init_per_group(_Name, Config) -> + Config. + +end_per_group(_, _) -> + ok. + +%% Generate a unique hostname for each testcase so that watchdogs +%% don't prevent a connection from being brought up immediately. +init_per_testcase(Name, Config) -> + Uniq = ["." ++ integer_to_list(N) || N <- tuple_to_list(now())], + [{host, lists:flatten([?L(Name) | Uniq])} | Config]. + +end_per_testcase(N, _) + when N == start; + N == start_services; + N == add_listeners; + N == remove_listeners; + N == stop_services; + N == stop -> + ok; +end_per_testcase(Name, Config) -> + CRef = ?util:read_priv(Config, Name), + ok = diameter:remove_transport(?CLIENT, CRef). + +%% Testcases all come in two flavours, client and server. +testcases() -> + lists:flatmap(fun tc/1, tc()). + +tc(Name) -> + [?A([C,$_|?L(Name)]) || C <- "cs"]. + +tc() -> + [no_common_application, + no_common_security, + unknown_peer, + unable, + client_reject]. + +%% =========================================================================== +%% start/stop testcases + +start(_Config) -> + ok = diameter:start(). + +start_services(_Config) -> + ok = diameter:start_service(?SERVER, ?SERVICE(?SERVER)), + ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT)). + +%% One server that responds only to base accounting, one that responds +%% to both this and the common application. Share a common service just +%% to simplify config, and because we can. +add_listeners(Config) -> + Acct = listen(?SERVER, + [{capabilities, [{'Origin-Host', ?HOST("acct-srv")}, + {'Auth-Application-Id', []}]}, + {applications, [accounting]}, + {capabilities_cb, [fun server_capx/3, acct]}]), + Base = listen(?SERVER, + [{capabilities, [{'Origin-Host', ?HOST("base-srv")}]}, + {capabilities_cb, [fun server_capx/3, base]}]), + ?util:write_priv(Config, ?MODULE, {Base, Acct}). %% lref/2 reads + +remove_listeners(_Config) -> + ok = diameter:remove_transport(?SERVER, true). + +stop_services(_Config) -> + ok = diameter:stop_service(?CLIENT), + ok = diameter:stop_service(?SERVER). + +stop(_Config) -> + ok = diameter:stop(). + +%% =========================================================================== +%% All the testcases come in pairs, one for receiving an event on the +%% client side, one on the server side. Note that testcases will +%% receive events resulting from other testcases when running in +%% parallel since the events are per service. The unique client +%% Origin-Host for each testcase plus transport references are used to +%% ensure that only the relevant event is extracted from the mailbox. +%% Don't bother extracting events that aren't relevant. + +%% ==================== +%% Ask the accounting server to speak the common application and expect +%% DIAMETER_NO_COMMON_APPLICATION = 5010. + +s_no_common_application(Config) -> + server_closed(Config, fun no_common_application/1, 5010). + +c_no_common_application(Config) -> + client_closed(Config, "acct-srv", fun no_common_application/1, 5010). + +no_common_application(Config) -> + connect(Config, acct, [{capabilities, [{'Acct-Application-Id', []}]}, + {applications, [common]}]). + +%% ==================== +%% Ask the base server to speak accounting with an unknown security +%% method and expect DIAMETER_NO_COMMON_SECURITY = 5017. + +s_no_common_security(Config) -> + server_closed(Config, fun no_common_security/1, 5017). + +c_no_common_security(Config) -> + client_closed(Config, "base-srv", fun no_common_security/1, 5017). + +no_common_security(Config) -> + connect(Config, base, [{capabilities, [{'Acct-Application-Id', []}, + {'Inband-Security-Id', [17, 18]}]}, + {applications, [common]}]). + +%% ==================== +%% Have the base server reject a decent CER with the protocol error +%% DIAMETER_UNKNOWN_PEER = 3010. + +s_unknown_peer(Config) -> + server_reject(Config, fun base/1, 3010). + +c_unknown_peer(Config) -> + true = diameter:subscribe(?CLIENT), + OH = ?HOST("base-srv"), + + {CRef, _} = base(Config), + + {'CEA', ?caps{}, + ?packet{msg = ?answer_message{'Origin-Host' = OH, + 'Result-Code' = 3010}}} + = client_recv(CRef). + +base(Config) -> + connect(Config, base, []). + +%% ==================== +%% Have the base server reject a decent CER with the non-protocol +%% error DIAMETER_UNABLE_TO_COMPLY = 5012. + +s_unable(Config) -> + server_reject(Config, fun base/1, 5012). + +c_unable(Config) -> + client_closed(Config, "base-srv", fun base/1, 5012). + +%% ==================== +%% Have the client reject a decent CEA. + +s_client_reject(Config) -> + true = diameter:subscribe(?SERVER), + OH = host(Config), + + {_, LRef} = client_reject(Config), + + receive + ?event{service = ?SERVER, + info = {up, LRef, + {_, ?caps{origin_host = {_, OH}}}, + {listen, _}, + ?packet{}}} + = Info -> + Info + after 2000 -> + fail({LRef, OH}) + end. + +c_client_reject(Config) -> + true = diameter:subscribe(?CLIENT), + OH = ?HOST("acct-srv"), + + {CRef, _} = client_reject(Config), + + {'CEA', {capabilities_cb, _, discard}, + ?caps{origin_host = {_, OH}}, + ?packet{msg = ?cea{'Result-Code' = 2001}}} + = client_recv(CRef). + +client_reject(Config) -> + connect(Config, acct, [{capabilities_cb, fun client_capx/2}]). + +%% =========================================================================== + +%% server_closed/3 + +server_closed(Config, F, RC) -> + true = diameter:subscribe(?SERVER), + OH = host(Config), + + {_, LRef} = F(Config), + + receive + ?event{service = ?SERVER, + info = {closed, LRef, + {'CER', RC, + ?caps{origin_host = {_, OH}}, + ?packet{}} + = Reason, + {listen, _}}} -> + Reason + after 2000 -> + fail({LRef, OH}) + end. + +%% server_reject/3 + +server_reject(Config, F, RC) -> + true = diameter:subscribe(?SERVER), + OH = host(Config), + + {_, LRef} = F(Config), + + receive + ?event{service = ?SERVER, + info = {closed, LRef, + {'CER', {capabilities_cb, _, RC}, + ?caps{origin_host = {_, OH}}, + ?packet{}} + = Reason, + {listen, _}}} -> + Reason + after 2000 -> + fail({LRef, OH}) + end. + +%% cliient_closed/4 + +client_closed(Config, Host, F, RC) -> + true = diameter:subscribe(?CLIENT), + OH = ?HOST(Host), + + {CRef, _} = F(Config), + + {'CEA', RC, ?caps{origin_host = {_, OH}}, ?packet{}} + = client_recv(CRef). + +%% client_recv/1 + +client_recv(CRef) -> + receive + ?event{service = ?CLIENT, + info = {closed, CRef, Reason, {connect, _}}} -> + Reason + after 2000 -> + fail(CRef) + end. + +%% server_capx/3 + +server_capx(_, ?caps{origin_host = {_, [_,$_|"unknown_peer." ++ _]}}, _) -> + unknown; + +server_capx(_, ?caps{origin_host = {_, [_,$_|"unable." ++ _]}}, _) -> + 5012; %% DIAMETER_UNABLE_TO_COMPLY + +server_capx(_, ?caps{origin_host = {OH,DH}}, _) -> + io:format("connection: ~p -> ~p~n", [DH,OH]), + ok. + +%% client_capx/2 + +client_capx(_, ?caps{origin_host = {[_,$_|"client_reject." ++ _], _}}) -> + discard. + +%% =========================================================================== + +fail(T) -> + erlang:error({T, process_info(self(), messages)}). + +host(Config) -> + {_, H} = lists:keyfind(host, 1, Config), + ?HOST(H). + +listen(Name, Opts) -> + ?util:listen(Name, tcp, Opts). + +connect(Config, T, Opts) -> + {_, H} = lists:keyfind(host, 1, Config), + LRef = lref(Config, T), + CRef = connect(LRef, [{capabilities, [{'Origin-Host', ?HOST(H)}]} + | Opts]), + Name = lists:takewhile(fun(C) -> C /= $. end, H), + ?util:write_priv(Config, Name, CRef), %% end_per_testcase reads + {CRef, LRef}. + +connect(LRef, Opts) -> + [PortNr] = ?util:lport(tcp, LRef, 20), + {ok, CRef} = diameter:add_transport(?CLIENT, + {connect, opts(PortNr, Opts)}), + CRef. + +opts(PortNr, Opts) -> + [{transport_module, diameter_tcp}, + {transport_config, [{raddr, ?ADDR}, + {rport, PortNr}, + {ip, ?ADDR}, + {port, 0}]} + | Opts]. + +lref(Config, T) -> + case ?util:read_priv(Config, ?MODULE) of + {LRef, _} when T == base -> + LRef; + {_, LRef} when T == acct -> + LRef + end. + +%% =========================================================================== +%% diameter callbacks + +peer_up(?SERVER, + {_, ?caps{origin_host = {"acct-srv." ++ _, + [_,$_|"client_reject." ++ _]}}}, + State, + _) -> + State. + +peer_down(?SERVER, + {_, ?caps{origin_host = {"acct-srv." ++ _, + [_,$_|"client_reject." ++ _]}}}, + State, + _) -> + State. diff --git a/lib/diameter/test/diameter_codec_test.erl b/lib/diameter/test/diameter_codec_test.erl index aab7ab35cc..8046ca4c04 100644 --- a/lib/diameter/test/diameter_codec_test.erl +++ b/lib/diameter/test/diameter_codec_test.erl @@ -25,7 +25,7 @@ %% Test encode/decode of dictionary-related modules. %% --include_lib("diameter/include/diameter.hrl"). +-include("diameter.hrl"). -define(BASE, diameter_gen_base_rfc3588). -define(BOOL, [true, false]). diff --git a/lib/diameter/test/diameter_failover_SUITE.erl b/lib/diameter/test/diameter_failover_SUITE.erl index c25e9682f0..f4d62f94c6 100644 --- a/lib/diameter/test/diameter_failover_SUITE.erl +++ b/lib/diameter/test/diameter_failover_SUITE.erl @@ -57,13 +57,8 @@ handle_error/4, handle_request/3]). --ifdef(DIAMETER_CT). +-include("diameter.hrl"). -include("diameter_gen_base_rfc3588.hrl"). --else. --include_lib("diameter/include/diameter_gen_base_rfc3588.hrl"). --endif. - --include_lib("diameter/include/diameter.hrl"). -include("diameter_ct.hrl"). %% =========================================================================== diff --git a/lib/diameter/test/diameter_relay_SUITE.erl b/lib/diameter/test/diameter_relay_SUITE.erl index 03f1115496..40cbdf805a 100644 --- a/lib/diameter/test/diameter_relay_SUITE.erl +++ b/lib/diameter/test/diameter_relay_SUITE.erl @@ -64,13 +64,8 @@ handle_error/4, handle_request/3]). --ifdef(DIAMETER_CT). +-include("diameter.hrl"). -include("diameter_gen_base_rfc3588.hrl"). --else. --include_lib("diameter/include/diameter_gen_base_rfc3588.hrl"). --endif. - --include_lib("diameter/include/diameter.hrl"). -include("diameter_ct.hrl"). %% =========================================================================== diff --git a/lib/diameter/test/diameter_tls_SUITE.erl b/lib/diameter/test/diameter_tls_SUITE.erl index 99f92ca0e0..127e3435dc 100644 --- a/lib/diameter/test/diameter_tls_SUITE.erl +++ b/lib/diameter/test/diameter_tls_SUITE.erl @@ -67,13 +67,8 @@ handle_error/4, handle_request/3]). --ifdef(DIAMETER_CT). +-include("diameter.hrl"). -include("diameter_gen_base_rfc3588.hrl"). --else. --include_lib("diameter/include/diameter_gen_base_rfc3588.hrl"). --endif. - --include_lib("diameter/include/diameter.hrl"). -include("diameter_ct.hrl"). %% =========================================================================== diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 6704f24532..55c5fc7c54 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -87,13 +87,8 @@ handle_error/5, handle_request/3]). --ifdef(DIAMETER_CT). +-include("diameter.hrl"). -include("diameter_gen_base_rfc3588.hrl"). --else. --include_lib("diameter/include/diameter_gen_base_rfc3588.hrl"). --endif. - --include_lib("diameter/include/diameter.hrl"). -include("diameter_ct.hrl"). %% =========================================================================== @@ -251,7 +246,7 @@ start_services(_Config) -> ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT)). add_transports(Config) -> - LRef = ?util:listen(?SERVER, tcp), + LRef = ?util:listen(?SERVER, tcp, [{capabilities_cb, fun capx/2}]), CRef = ?util:connect(?CLIENT, tcp, LRef), ?util:write_priv(Config, "transport", {LRef, CRef}). @@ -266,6 +261,10 @@ stop_services(_Config) -> stop(_Config) -> ok = diameter:stop(). +capx(_, #diameter_caps{origin_host = {OH,DH}}) -> + io:format("connection: ~p -> ~p~n", [DH,OH]), + ok. + %% =========================================================================== %% Ensure that result codes have the expected values. diff --git a/lib/diameter/test/diameter_transport_SUITE.erl b/lib/diameter/test/diameter_transport_SUITE.erl index a9520ef5bd..c22adc3334 100644 --- a/lib/diameter/test/diameter_transport_SUITE.erl +++ b/lib/diameter/test/diameter_transport_SUITE.erl @@ -45,7 +45,7 @@ init/2]). -include_lib("kernel/include/inet_sctp.hrl"). --include_lib("diameter/include/diameter.hrl"). +-include("diameter.hrl"). -include("diameter_ct.hrl"). -define(util, diameter_util). diff --git a/lib/diameter/test/diameter_util.erl b/lib/diameter/test/diameter_util.erl index 3fe8ea5363..6b1dc1f0c9 100644 --- a/lib/diameter/test/diameter_util.erl +++ b/lib/diameter/test/diameter_util.erl @@ -33,8 +33,8 @@ %% diameter-specific -export([lport/2, lport/3, - listen/2, - connect/3, + listen/2, listen/3, + connect/3, connect/4, disconnect/4]). %% common_test-specific @@ -254,22 +254,28 @@ lp(M, Ref, T) -> end. %% --------------------------------------------------------------------------- -%% listen/2 +%% listen/2-3 %% %% Add a listening transport on the loopback address and a free port. listen(SvcName, Prot) -> - add_transport(SvcName, {listen, opts(Prot, listen)}). + listen(SvcName, Prot, []). + +listen(SvcName, Prot, Opts) -> + add_transport(SvcName, {listen, opts(Prot, listen) ++ Opts}). %% --------------------------------------------------------------------------- -%% connect/3 +%% connect/2-3 %% %% Add a connecting transport on and connect to a listening transport %% with the specified reference. connect(Client, Prot, LRef) -> + connect(Client, Prot, LRef, []). + +connect(Client, Prot, LRef, Opts) -> [PortNr] = lport(Prot, LRef, 20), - Ref = add_transport(Client, {connect, opts(Prot, PortNr)}), + Ref = add_transport(Client, {connect, opts(Prot, PortNr) ++ Opts}), true = diameter:subscribe(Client), ok = receive {diameter_event, Client, {up, Ref, _, _, _}} -> ok diff --git a/lib/diameter/test/diameter_watchdog_SUITE.erl b/lib/diameter/test/diameter_watchdog_SUITE.erl index dec307529a..b40d7c104d 100644 --- a/lib/diameter/test/diameter_watchdog_SUITE.erl +++ b/lib/diameter/test/diameter_watchdog_SUITE.erl @@ -36,7 +36,7 @@ id/1, %% jitter callback run/1]). --include_lib("diameter/include/diameter.hrl"). +-include("diameter.hrl"). -include("diameter_ct.hrl"). %% =========================================================================== diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk index 75fdd0bd1d..f88258c232 100644 --- a/lib/diameter/test/modules.mk +++ b/lib/diameter/test/modules.mk @@ -33,6 +33,7 @@ MODULES = \ diameter_stats_SUITE \ diameter_watchdog_SUITE \ diameter_transport_SUITE \ + diameter_capx_SUITE \ diameter_traffic_SUITE \ diameter_relay_SUITE \ diameter_tls_SUITE \ diff --git a/lib/diameter/test/release.sed b/lib/diameter/test/release.sed new file mode 100644 index 0000000000..2720b778f2 --- /dev/null +++ b/lib/diameter/test/release.sed @@ -0,0 +1,35 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2010-2011. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +# +# This bit of gymnastics is to replace the include of diameter's +# public hrls by include_lib when releasing testsuites, so that they +# compile both in the development filesystem (where generated hrls +# aren't in diameter/include) and with common_test's autocompilation +# on an installed release. Solving the problem by installing generated +# hrls to ../include is anathema: that directory is for handwritten +# source.) +# + +/^-include("/!b +/"diameter_gen_/b s +/"diameter\./!b + +:s +s@("@_lib&diameter/include/@ diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index c783450c9f..b1d3ba2241 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 0.10 +DIAMETER_VSN = 0.11 PRE_VSN = APP_VSN = "$(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN)" diff --git a/lib/docbuilder/test/docb_SUITE_data/cdata_problem.xml b/lib/docbuilder/test/docb_SUITE_data/cdata_problem.xml index b7f6f5376e..b7f6f5376e 100755..100644 --- a/lib/docbuilder/test/docb_SUITE_data/cdata_problem.xml +++ b/lib/docbuilder/test/docb_SUITE_data/cdata_problem.xml diff --git a/lib/docbuilder/xsd/application.xsd b/lib/docbuilder/xsd/application.xsd index eb666cb6c7..eb666cb6c7 100755..100644 --- a/lib/docbuilder/xsd/application.xsd +++ b/lib/docbuilder/xsd/application.xsd diff --git a/lib/docbuilder/xsd/appref.xsd b/lib/docbuilder/xsd/appref.xsd index b63839e494..b63839e494 100755..100644 --- a/lib/docbuilder/xsd/appref.xsd +++ b/lib/docbuilder/xsd/appref.xsd diff --git a/lib/docbuilder/xsd/book.xsd b/lib/docbuilder/xsd/book.xsd index b47962263a..b47962263a 100755..100644 --- a/lib/docbuilder/xsd/book.xsd +++ b/lib/docbuilder/xsd/book.xsd diff --git a/lib/docbuilder/xsd/chapter.xsd b/lib/docbuilder/xsd/chapter.xsd index 4d89baa988..4d89baa988 100755..100644 --- a/lib/docbuilder/xsd/chapter.xsd +++ b/lib/docbuilder/xsd/chapter.xsd diff --git a/lib/docbuilder/xsd/common.entities.xsd b/lib/docbuilder/xsd/common.entities.xsd index 52a5d35179..52a5d35179 100755..100644 --- a/lib/docbuilder/xsd/common.entities.xsd +++ b/lib/docbuilder/xsd/common.entities.xsd diff --git a/lib/docbuilder/xsd/common.header.xsd b/lib/docbuilder/xsd/common.header.xsd index bfee4b8bb4..bfee4b8bb4 100755..100644 --- a/lib/docbuilder/xsd/common.header.xsd +++ b/lib/docbuilder/xsd/common.header.xsd diff --git a/lib/docbuilder/xsd/common.image.xsd b/lib/docbuilder/xsd/common.image.xsd index 17054eb23c..17054eb23c 100755..100644 --- a/lib/docbuilder/xsd/common.image.xsd +++ b/lib/docbuilder/xsd/common.image.xsd diff --git a/lib/docbuilder/xsd/common.refs.xsd b/lib/docbuilder/xsd/common.refs.xsd index 58b450669d..58b450669d 100755..100644 --- a/lib/docbuilder/xsd/common.refs.xsd +++ b/lib/docbuilder/xsd/common.refs.xsd diff --git a/lib/docbuilder/xsd/common.table.xsd b/lib/docbuilder/xsd/common.table.xsd index cf63df4317..cf63df4317 100755..100644 --- a/lib/docbuilder/xsd/common.table.xsd +++ b/lib/docbuilder/xsd/common.table.xsd diff --git a/lib/docbuilder/xsd/common.xsd b/lib/docbuilder/xsd/common.xsd index 3d43390bd8..3d43390bd8 100755..100644 --- a/lib/docbuilder/xsd/common.xsd +++ b/lib/docbuilder/xsd/common.xsd diff --git a/lib/docbuilder/xsd/comref.xsd b/lib/docbuilder/xsd/comref.xsd index 61df4dd848..61df4dd848 100755..100644 --- a/lib/docbuilder/xsd/comref.xsd +++ b/lib/docbuilder/xsd/comref.xsd diff --git a/lib/docbuilder/xsd/cref.xsd b/lib/docbuilder/xsd/cref.xsd index f1cbeddfff..f1cbeddfff 100755..100644 --- a/lib/docbuilder/xsd/cref.xsd +++ b/lib/docbuilder/xsd/cref.xsd diff --git a/lib/docbuilder/xsd/erlref.xsd b/lib/docbuilder/xsd/erlref.xsd index f6011b7bea..f6011b7bea 100755..100644 --- a/lib/docbuilder/xsd/erlref.xsd +++ b/lib/docbuilder/xsd/erlref.xsd diff --git a/lib/docbuilder/xsd/fascicules.xsd b/lib/docbuilder/xsd/fascicules.xsd index bfdb5bd604..bfdb5bd604 100755..100644 --- a/lib/docbuilder/xsd/fascicules.xsd +++ b/lib/docbuilder/xsd/fascicules.xsd diff --git a/lib/docbuilder/xsd/fileref.xsd b/lib/docbuilder/xsd/fileref.xsd index 8038f2115f..8038f2115f 100755..100644 --- a/lib/docbuilder/xsd/fileref.xsd +++ b/lib/docbuilder/xsd/fileref.xsd diff --git a/lib/docbuilder/xsd/part.xsd b/lib/docbuilder/xsd/part.xsd index 30d6ec0120..30d6ec0120 100755..100644 --- a/lib/docbuilder/xsd/part.xsd +++ b/lib/docbuilder/xsd/part.xsd diff --git a/lib/hipe/icode/hipe_icode_pp.erl b/lib/hipe/icode/hipe_icode_pp.erl index 575bbfe43d..575bbfe43d 100755..100644 --- a/lib/hipe/icode/hipe_icode_pp.erl +++ b/lib/hipe/icode/hipe_icode_pp.erl diff --git a/lib/hipe/icode/hipe_icode_ssa.erl b/lib/hipe/icode/hipe_icode_ssa.erl index 719d5d8f45..719d5d8f45 100755..100644 --- a/lib/hipe/icode/hipe_icode_ssa.erl +++ b/lib/hipe/icode/hipe_icode_ssa.erl diff --git a/lib/hipe/util/hipe_dot.erl b/lib/hipe/util/hipe_dot.erl index d6ef801c88..d6ef801c88 100755..100644 --- a/lib/hipe/util/hipe_dot.erl +++ b/lib/hipe/util/hipe_dot.erl diff --git a/lib/inets/doc/archive/rfc3986.txt b/lib/inets/doc/archive/rfc3986.txt new file mode 100644 index 0000000000..c56ed4eb70 --- /dev/null +++ b/lib/inets/doc/archive/rfc3986.txt @@ -0,0 +1,3419 @@ + + + + + + +Network Working Group T. Berners-Lee +Request for Comments: 3986 W3C/MIT +STD: 66 R. Fielding +Updates: 1738 Day Software +Obsoletes: 2732, 2396, 1808 L. Masinter +Category: Standards Track Adobe Systems + January 2005 + + + Uniform Resource Identifier (URI): Generic Syntax + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2005). + +Abstract + + A Uniform Resource Identifier (URI) is a compact sequence of + characters that identifies an abstract or physical resource. This + specification defines the generic URI syntax and a process for + resolving URI references that might be in relative form, along with + guidelines and security considerations for the use of URIs on the + Internet. The URI syntax defines a grammar that is a superset of all + valid URIs, allowing an implementation to parse the common components + of a URI reference without knowing the scheme-specific requirements + of every possible identifier. This specification does not define a + generative grammar for URIs; that task is performed by the individual + specifications of each URI scheme. + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 1] + +RFC 3986 URI Generic Syntax January 2005 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 1.1. Overview of URIs . . . . . . . . . . . . . . . . . . . . 4 + 1.1.1. Generic Syntax . . . . . . . . . . . . . . . . . 6 + 1.1.2. Examples . . . . . . . . . . . . . . . . . . . . 7 + 1.1.3. URI, URL, and URN . . . . . . . . . . . . . . . 7 + 1.2. Design Considerations . . . . . . . . . . . . . . . . . 8 + 1.2.1. Transcription . . . . . . . . . . . . . . . . . 8 + 1.2.2. Separating Identification from Interaction . . . 9 + 1.2.3. Hierarchical Identifiers . . . . . . . . . . . . 10 + 1.3. Syntax Notation . . . . . . . . . . . . . . . . . . . . 11 + 2. Characters . . . . . . . . . . . . . . . . . . . . . . . . . . 11 + 2.1. Percent-Encoding . . . . . . . . . . . . . . . . . . . . 12 + 2.2. Reserved Characters . . . . . . . . . . . . . . . . . . 12 + 2.3. Unreserved Characters . . . . . . . . . . . . . . . . . 13 + 2.4. When to Encode or Decode . . . . . . . . . . . . . . . . 14 + 2.5. Identifying Data . . . . . . . . . . . . . . . . . . . . 14 + 3. Syntax Components . . . . . . . . . . . . . . . . . . . . . . 16 + 3.1. Scheme . . . . . . . . . . . . . . . . . . . . . . . . . 17 + 3.2. Authority . . . . . . . . . . . . . . . . . . . . . . . 17 + 3.2.1. User Information . . . . . . . . . . . . . . . . 18 + 3.2.2. Host . . . . . . . . . . . . . . . . . . . . . . 18 + 3.2.3. Port . . . . . . . . . . . . . . . . . . . . . . 22 + 3.3. Path . . . . . . . . . . . . . . . . . . . . . . . . . . 22 + 3.4. Query . . . . . . . . . . . . . . . . . . . . . . . . . 23 + 3.5. Fragment . . . . . . . . . . . . . . . . . . . . . . . . 24 + 4. Usage . . . . . . . . . . . . . . . . . . . . . . . . . . . . 25 + 4.1. URI Reference . . . . . . . . . . . . . . . . . . . . . 25 + 4.2. Relative Reference . . . . . . . . . . . . . . . . . . . 26 + 4.3. Absolute URI . . . . . . . . . . . . . . . . . . . . . . 27 + 4.4. Same-Document Reference . . . . . . . . . . . . . . . . 27 + 4.5. Suffix Reference . . . . . . . . . . . . . . . . . . . . 27 + 5. Reference Resolution . . . . . . . . . . . . . . . . . . . . . 28 + 5.1. Establishing a Base URI . . . . . . . . . . . . . . . . 28 + 5.1.1. Base URI Embedded in Content . . . . . . . . . . 29 + 5.1.2. Base URI from the Encapsulating Entity . . . . . 29 + 5.1.3. Base URI from the Retrieval URI . . . . . . . . 30 + 5.1.4. Default Base URI . . . . . . . . . . . . . . . . 30 + 5.2. Relative Resolution . . . . . . . . . . . . . . . . . . 30 + 5.2.1. Pre-parse the Base URI . . . . . . . . . . . . . 31 + 5.2.2. Transform References . . . . . . . . . . . . . . 31 + 5.2.3. Merge Paths . . . . . . . . . . . . . . . . . . 32 + 5.2.4. Remove Dot Segments . . . . . . . . . . . . . . 33 + 5.3. Component Recomposition . . . . . . . . . . . . . . . . 35 + 5.4. Reference Resolution Examples . . . . . . . . . . . . . 35 + 5.4.1. Normal Examples . . . . . . . . . . . . . . . . 36 + 5.4.2. Abnormal Examples . . . . . . . . . . . . . . . 36 + + + +Berners-Lee, et al. Standards Track [Page 2] + +RFC 3986 URI Generic Syntax January 2005 + + + 6. Normalization and Comparison . . . . . . . . . . . . . . . . . 38 + 6.1. Equivalence . . . . . . . . . . . . . . . . . . . . . . 38 + 6.2. Comparison Ladder . . . . . . . . . . . . . . . . . . . 39 + 6.2.1. Simple String Comparison . . . . . . . . . . . . 39 + 6.2.2. Syntax-Based Normalization . . . . . . . . . . . 40 + 6.2.3. Scheme-Based Normalization . . . . . . . . . . . 41 + 6.2.4. Protocol-Based Normalization . . . . . . . . . . 42 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 43 + 7.1. Reliability and Consistency . . . . . . . . . . . . . . 43 + 7.2. Malicious Construction . . . . . . . . . . . . . . . . . 43 + 7.3. Back-End Transcoding . . . . . . . . . . . . . . . . . . 44 + 7.4. Rare IP Address Formats . . . . . . . . . . . . . . . . 45 + 7.5. Sensitive Information . . . . . . . . . . . . . . . . . 45 + 7.6. Semantic Attacks . . . . . . . . . . . . . . . . . . . . 45 + 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 46 + 9. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . . 46 + 10. References . . . . . . . . . . . . . . . . . . . . . . . . . . 46 + 10.1. Normative References . . . . . . . . . . . . . . . . . . 46 + 10.2. Informative References . . . . . . . . . . . . . . . . . 47 + A. Collected ABNF for URI . . . . . . . . . . . . . . . . . . . . 49 + B. Parsing a URI Reference with a Regular Expression . . . . . . 50 + C. Delimiting a URI in Context . . . . . . . . . . . . . . . . . 51 + D. Changes from RFC 2396 . . . . . . . . . . . . . . . . . . . . 53 + D.1. Additions . . . . . . . . . . . . . . . . . . . . . . . 53 + D.2. Modifications . . . . . . . . . . . . . . . . . . . . . 53 + Index . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 56 + Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . . 60 + Full Copyright Statement . . . . . . . . . . . . . . . . . . . . . 61 + + + + + + + + + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 3] + +RFC 3986 URI Generic Syntax January 2005 + + +1. Introduction + + A Uniform Resource Identifier (URI) provides a simple and extensible + means for identifying a resource. This specification of URI syntax + and semantics is derived from concepts introduced by the World Wide + Web global information initiative, whose use of these identifiers + dates from 1990 and is described in "Universal Resource Identifiers + in WWW" [RFC1630]. The syntax is designed to meet the + recommendations laid out in "Functional Recommendations for Internet + Resource Locators" [RFC1736] and "Functional Requirements for Uniform + Resource Names" [RFC1737]. + + This document obsoletes [RFC2396], which merged "Uniform Resource + Locators" [RFC1738] and "Relative Uniform Resource Locators" + [RFC1808] in order to define a single, generic syntax for all URIs. + It obsoletes [RFC2732], which introduced syntax for an IPv6 address. + It excludes portions of RFC 1738 that defined the specific syntax of + individual URI schemes; those portions will be updated as separate + documents. The process for registration of new URI schemes is + defined separately by [BCP35]. Advice for designers of new URI + schemes can be found in [RFC2718]. All significant changes from RFC + 2396 are noted in Appendix D. + + This specification uses the terms "character" and "coded character + set" in accordance with the definitions provided in [BCP19], and + "character encoding" in place of what [BCP19] refers to as a + "charset". + +1.1. Overview of URIs + + URIs are characterized as follows: + + Uniform + + Uniformity provides several benefits. It allows different types + of resource identifiers to be used in the same context, even when + the mechanisms used to access those resources may differ. It + allows uniform semantic interpretation of common syntactic + conventions across different types of resource identifiers. It + allows introduction of new types of resource identifiers without + interfering with the way that existing identifiers are used. It + allows the identifiers to be reused in many different contexts, + thus permitting new applications or protocols to leverage a pre- + existing, large, and widely used set of resource identifiers. + + + + + + + +Berners-Lee, et al. Standards Track [Page 4] + +RFC 3986 URI Generic Syntax January 2005 + + + Resource + + This specification does not limit the scope of what might be a + resource; rather, the term "resource" is used in a general sense + for whatever might be identified by a URI. Familiar examples + include an electronic document, an image, a source of information + with a consistent purpose (e.g., "today's weather report for Los + Angeles"), a service (e.g., an HTTP-to-SMS gateway), and a + collection of other resources. A resource is not necessarily + accessible via the Internet; e.g., human beings, corporations, and + bound books in a library can also be resources. Likewise, + abstract concepts can be resources, such as the operators and + operands of a mathematical equation, the types of a relationship + (e.g., "parent" or "employee"), or numeric values (e.g., zero, + one, and infinity). + + Identifier + + An identifier embodies the information required to distinguish + what is being identified from all other things within its scope of + identification. Our use of the terms "identify" and "identifying" + refer to this purpose of distinguishing one resource from all + other resources, regardless of how that purpose is accomplished + (e.g., by name, address, or context). These terms should not be + mistaken as an assumption that an identifier defines or embodies + the identity of what is referenced, though that may be the case + for some identifiers. Nor should it be assumed that a system + using URIs will access the resource identified: in many cases, + URIs are used to denote resources without any intention that they + be accessed. Likewise, the "one" resource identified might not be + singular in nature (e.g., a resource might be a named set or a + mapping that varies over time). + + A URI is an identifier consisting of a sequence of characters + matching the syntax rule named <URI> in Section 3. It enables + uniform identification of resources via a separately defined + extensible set of naming schemes (Section 3.1). How that + identification is accomplished, assigned, or enabled is delegated to + each scheme specification. + + This specification does not place any limits on the nature of a + resource, the reasons why an application might seek to refer to a + resource, or the kinds of systems that might use URIs for the sake of + identifying resources. This specification does not require that a + URI persists in identifying the same resource over time, though that + is a common goal of all URI schemes. Nevertheless, nothing in this + + + + + +Berners-Lee, et al. Standards Track [Page 5] + +RFC 3986 URI Generic Syntax January 2005 + + + specification prevents an application from limiting itself to + particular types of resources, or to a subset of URIs that maintains + characteristics desired by that application. + + URIs have a global scope and are interpreted consistently regardless + of context, though the result of that interpretation may be in + relation to the end-user's context. For example, "http://localhost/" + has the same interpretation for every user of that reference, even + though the network interface corresponding to "localhost" may be + different for each end-user: interpretation is independent of access. + However, an action made on the basis of that reference will take + place in relation to the end-user's context, which implies that an + action intended to refer to a globally unique thing must use a URI + that distinguishes that resource from all other things. URIs that + identify in relation to the end-user's local context should only be + used when the context itself is a defining aspect of the resource, + such as when an on-line help manual refers to a file on the end- + user's file system (e.g., "file:///etc/hosts"). + +1.1.1. Generic Syntax + + Each URI begins with a scheme name, as defined in Section 3.1, that + refers to a specification for assigning identifiers within that + scheme. As such, the URI syntax is a federated and extensible naming + system wherein each scheme's specification may further restrict the + syntax and semantics of identifiers using that scheme. + + This specification defines those elements of the URI syntax that are + required of all URI schemes or are common to many URI schemes. It + thus defines the syntax and semantics needed to implement a scheme- + independent parsing mechanism for URI references, by which the + scheme-dependent handling of a URI can be postponed until the + scheme-dependent semantics are needed. Likewise, protocols and data + formats that make use of URI references can refer to this + specification as a definition for the range of syntax allowed for all + URIs, including those schemes that have yet to be defined. This + decouples the evolution of identification schemes from the evolution + of protocols, data formats, and implementations that make use of + URIs. + + A parser of the generic URI syntax can parse any URI reference into + its major components. Once the scheme is determined, further + scheme-specific parsing can be performed on the components. In other + words, the URI generic syntax is a superset of the syntax of all URI + schemes. + + + + + + +Berners-Lee, et al. Standards Track [Page 6] + +RFC 3986 URI Generic Syntax January 2005 + + +1.1.2. Examples + + The following example URIs illustrate several URI schemes and + variations in their common syntax components: + + ftp://ftp.is.co.za/rfc/rfc1808.txt + + http://www.ietf.org/rfc/rfc2396.txt + + ldap://[2001:db8::7]/c=GB?objectClass?one + + mailto:[email protected] + + news:comp.infosystems.www.servers.unix + + tel:+1-816-555-1212 + + telnet://192.0.2.16:80/ + + urn:oasis:names:specification:docbook:dtd:xml:4.1.2 + + +1.1.3. URI, URL, and URN + + A URI can be further classified as a locator, a name, or both. The + term "Uniform Resource Locator" (URL) refers to the subset of URIs + that, in addition to identifying a resource, provide a means of + locating the resource by describing its primary access mechanism + (e.g., its network "location"). The term "Uniform Resource Name" + (URN) has been used historically to refer to both URIs under the + "urn" scheme [RFC2141], which are required to remain globally unique + and persistent even when the resource ceases to exist or becomes + unavailable, and to any other URI with the properties of a name. + + An individual scheme does not have to be classified as being just one + of "name" or "locator". Instances of URIs from any given scheme may + have the characteristics of names or locators or both, often + depending on the persistence and care in the assignment of + identifiers by the naming authority, rather than on any quality of + the scheme. Future specifications and related documentation should + use the general term "URI" rather than the more restrictive terms + "URL" and "URN" [RFC3305]. + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 7] + +RFC 3986 URI Generic Syntax January 2005 + + +1.2. Design Considerations + +1.2.1. Transcription + + The URI syntax has been designed with global transcription as one of + its main considerations. A URI is a sequence of characters from a + very limited set: the letters of the basic Latin alphabet, digits, + and a few special characters. A URI may be represented in a variety + of ways; e.g., ink on paper, pixels on a screen, or a sequence of + character encoding octets. The interpretation of a URI depends only + on the characters used and not on how those characters are + represented in a network protocol. + + The goal of transcription can be described by a simple scenario. + Imagine two colleagues, Sam and Kim, sitting in a pub at an + international conference and exchanging research ideas. Sam asks Kim + for a location to get more information, so Kim writes the URI for the + research site on a napkin. Upon returning home, Sam takes out the + napkin and types the URI into a computer, which then retrieves the + information to which Kim referred. + + There are several design considerations revealed by the scenario: + + o A URI is a sequence of characters that is not always represented + as a sequence of octets. + + o A URI might be transcribed from a non-network source and thus + should consist of characters that are most likely able to be + entered into a computer, within the constraints imposed by + keyboards (and related input devices) across languages and + locales. + + o A URI often has to be remembered by people, and it is easier for + people to remember a URI when it consists of meaningful or + familiar components. + + These design considerations are not always in alignment. For + example, it is often the case that the most meaningful name for a URI + component would require characters that cannot be typed into some + systems. The ability to transcribe a resource identifier from one + medium to another has been considered more important than having a + URI consist of the most meaningful of components. + + In local or regional contexts and with improving technology, users + might benefit from being able to use a wider range of characters; + such use is not defined by this specification. Percent-encoded + octets (Section 2.1) may be used within a URI to represent characters + outside the range of the US-ASCII coded character set if this + + + +Berners-Lee, et al. Standards Track [Page 8] + +RFC 3986 URI Generic Syntax January 2005 + + + representation is allowed by the scheme or by the protocol element in + which the URI is referenced. Such a definition should specify the + character encoding used to map those characters to octets prior to + being percent-encoded for the URI. + +1.2.2. Separating Identification from Interaction + + A common misunderstanding of URIs is that they are only used to refer + to accessible resources. The URI itself only provides + identification; access to the resource is neither guaranteed nor + implied by the presence of a URI. Instead, any operation associated + with a URI reference is defined by the protocol element, data format + attribute, or natural language text in which it appears. + + Given a URI, a system may attempt to perform a variety of operations + on the resource, as might be characterized by words such as "access", + "update", "replace", or "find attributes". Such operations are + defined by the protocols that make use of URIs, not by this + specification. However, we do use a few general terms for describing + common operations on URIs. URI "resolution" is the process of + determining an access mechanism and the appropriate parameters + necessary to dereference a URI; this resolution may require several + iterations. To use that access mechanism to perform an action on the + URI's resource is to "dereference" the URI. + + When URIs are used within information retrieval systems to identify + sources of information, the most common form of URI dereference is + "retrieval": making use of a URI in order to retrieve a + representation of its associated resource. A "representation" is a + sequence of octets, along with representation metadata describing + those octets, that constitutes a record of the state of the resource + at the time when the representation is generated. Retrieval is + achieved by a process that might include using the URI as a cache key + to check for a locally cached representation, resolution of the URI + to determine an appropriate access mechanism (if any), and + dereference of the URI for the sake of applying a retrieval + operation. Depending on the protocols used to perform the retrieval, + additional information might be supplied about the resource (resource + metadata) and its relation to other resources. + + URI references in information retrieval systems are designed to be + late-binding: the result of an access is generally determined when it + is accessed and may vary over time or due to other aspects of the + interaction. These references are created in order to be used in the + future: what is being identified is not some specific result that was + obtained in the past, but rather some characteristic that is expected + to be true for future results. In such cases, the resource referred + to by the URI is actually a sameness of characteristics as observed + + + +Berners-Lee, et al. Standards Track [Page 9] + +RFC 3986 URI Generic Syntax January 2005 + + + over time, perhaps elucidated by additional comments or assertions + made by the resource provider. + + Although many URI schemes are named after protocols, this does not + imply that use of these URIs will result in access to the resource + via the named protocol. URIs are often used simply for the sake of + identification. Even when a URI is used to retrieve a representation + of a resource, that access might be through gateways, proxies, + caches, and name resolution services that are independent of the + protocol associated with the scheme name. The resolution of some + URIs may require the use of more than one protocol (e.g., both DNS + and HTTP are typically used to access an "http" URI's origin server + when a representation isn't found in a local cache). + +1.2.3. Hierarchical Identifiers + + The URI syntax is organized hierarchically, with components listed in + order of decreasing significance from left to right. For some URI + schemes, the visible hierarchy is limited to the scheme itself: + everything after the scheme component delimiter (":") is considered + opaque to URI processing. Other URI schemes make the hierarchy + explicit and visible to generic parsing algorithms. + + The generic syntax uses the slash ("/"), question mark ("?"), and + number sign ("#") characters to delimit components that are + significant to the generic parser's hierarchical interpretation of an + identifier. In addition to aiding the readability of such + identifiers through the consistent use of familiar syntax, this + uniform representation of hierarchy across naming schemes allows + scheme-independent references to be made relative to that hierarchy. + + It is often the case that a group or "tree" of documents has been + constructed to serve a common purpose, wherein the vast majority of + URI references in these documents point to resources within the tree + rather than outside it. Similarly, documents located at a particular + site are much more likely to refer to other resources at that site + than to resources at remote sites. Relative referencing of URIs + allows document trees to be partially independent of their location + and access scheme. For instance, it is possible for a single set of + hypertext documents to be simultaneously accessible and traversable + via each of the "file", "http", and "ftp" schemes if the documents + refer to each other with relative references. Furthermore, such + document trees can be moved, as a whole, without changing any of the + relative references. + + A relative reference (Section 4.2) refers to a resource by describing + the difference within a hierarchical name space between the reference + context and the target URI. The reference resolution algorithm, + + + +Berners-Lee, et al. Standards Track [Page 10] + +RFC 3986 URI Generic Syntax January 2005 + + + presented in Section 5, defines how such a reference is transformed + to the target URI. As relative references can only be used within + the context of a hierarchical URI, designers of new URI schemes + should use a syntax consistent with the generic syntax's hierarchical + components unless there are compelling reasons to forbid relative + referencing within that scheme. + + NOTE: Previous specifications used the terms "partial URI" and + "relative URI" to denote a relative reference to a URI. As some + readers misunderstood those terms to mean that relative URIs are a + subset of URIs rather than a method of referencing URIs, this + specification simply refers to them as relative references. + + All URI references are parsed by generic syntax parsers when used. + However, because hierarchical processing has no effect on an absolute + URI used in a reference unless it contains one or more dot-segments + (complete path segments of "." or "..", as described in Section 3.3), + URI scheme specifications can define opaque identifiers by + disallowing use of slash characters, question mark characters, and + the URIs "scheme:." and "scheme:..". + +1.3. Syntax Notation + + This specification uses the Augmented Backus-Naur Form (ABNF) + notation of [RFC2234], including the following core ABNF syntax rules + defined by that specification: ALPHA (letters), CR (carriage return), + DIGIT (decimal digits), DQUOTE (double quote), HEXDIG (hexadecimal + digits), LF (line feed), and SP (space). The complete URI syntax is + collected in Appendix A. + +2. Characters + + The URI syntax provides a method of encoding data, presumably for the + sake of identifying a resource, as a sequence of characters. The URI + characters are, in turn, frequently encoded as octets for transport + or presentation. This specification does not mandate any particular + character encoding for mapping between URI characters and the octets + used to store or transmit those characters. When a URI appears in a + protocol element, the character encoding is defined by that protocol; + without such a definition, a URI is assumed to be in the same + character encoding as the surrounding text. + + The ABNF notation defines its terminal values to be non-negative + integers (codepoints) based on the US-ASCII coded character set + [ASCII]. Because a URI is a sequence of characters, we must invert + that relation in order to understand the URI syntax. Therefore, the + + + + + +Berners-Lee, et al. Standards Track [Page 11] + +RFC 3986 URI Generic Syntax January 2005 + + + integer values used by the ABNF must be mapped back to their + corresponding characters via US-ASCII in order to complete the syntax + rules. + + A URI is composed from a limited set of characters consisting of + digits, letters, and a few graphic symbols. A reserved subset of + those characters may be used to delimit syntax components within a + URI while the remaining characters, including both the unreserved set + and those reserved characters not acting as delimiters, define each + component's identifying data. + +2.1. Percent-Encoding + + A percent-encoding mechanism is used to represent a data octet in a + component when that octet's corresponding character is outside the + allowed set or is being used as a delimiter of, or within, the + component. A percent-encoded octet is encoded as a character + triplet, consisting of the percent character "%" followed by the two + hexadecimal digits representing that octet's numeric value. For + example, "%20" is the percent-encoding for the binary octet + "00100000" (ABNF: %x20), which in US-ASCII corresponds to the space + character (SP). Section 2.4 describes when percent-encoding and + decoding is applied. + + pct-encoded = "%" HEXDIG HEXDIG + + The uppercase hexadecimal digits 'A' through 'F' are equivalent to + the lowercase digits 'a' through 'f', respectively. If two URIs + differ only in the case of hexadecimal digits used in percent-encoded + octets, they are equivalent. For consistency, URI producers and + normalizers should use uppercase hexadecimal digits for all percent- + encodings. + +2.2. Reserved Characters + + URIs include components and subcomponents that are delimited by + characters in the "reserved" set. These characters are called + "reserved" because they may (or may not) be defined as delimiters by + the generic syntax, by each scheme-specific syntax, or by the + implementation-specific syntax of a URI's dereferencing algorithm. + If data for a URI component would conflict with a reserved + character's purpose as a delimiter, then the conflicting data must be + percent-encoded before the URI is formed. + + + + + + + + +Berners-Lee, et al. Standards Track [Page 12] + +RFC 3986 URI Generic Syntax January 2005 + + + reserved = gen-delims / sub-delims + + gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + + sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + / "*" / "+" / "," / ";" / "=" + + The purpose of reserved characters is to provide a set of delimiting + characters that are distinguishable from other data within a URI. + URIs that differ in the replacement of a reserved character with its + corresponding percent-encoded octet are not equivalent. Percent- + encoding a reserved character, or decoding a percent-encoded octet + that corresponds to a reserved character, will change how the URI is + interpreted by most applications. Thus, characters in the reserved + set are protected from normalization and are therefore safe to be + used by scheme-specific and producer-specific algorithms for + delimiting data subcomponents within a URI. + + A subset of the reserved characters (gen-delims) is used as + delimiters of the generic URI components described in Section 3. A + component's ABNF syntax rule will not use the reserved or gen-delims + rule names directly; instead, each syntax rule lists the characters + allowed within that component (i.e., not delimiting it), and any of + those characters that are also in the reserved set are "reserved" for + use as subcomponent delimiters within the component. Only the most + common subcomponents are defined by this specification; other + subcomponents may be defined by a URI scheme's specification, or by + the implementation-specific syntax of a URI's dereferencing + algorithm, provided that such subcomponents are delimited by + characters in the reserved set allowed within that component. + + URI producing applications should percent-encode data octets that + correspond to characters in the reserved set unless these characters + are specifically allowed by the URI scheme to represent data in that + component. If a reserved character is found in a URI component and + no delimiting role is known for that character, then it must be + interpreted as representing the data octet corresponding to that + character's encoding in US-ASCII. + +2.3. Unreserved Characters + + Characters that are allowed in a URI but do not have a reserved + purpose are called unreserved. These include uppercase and lowercase + letters, decimal digits, hyphen, period, underscore, and tilde. + + unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + + + + + +Berners-Lee, et al. Standards Track [Page 13] + +RFC 3986 URI Generic Syntax January 2005 + + + URIs that differ in the replacement of an unreserved character with + its corresponding percent-encoded US-ASCII octet are equivalent: they + identify the same resource. However, URI comparison implementations + do not always perform normalization prior to comparison (see Section + 6). For consistency, percent-encoded octets in the ranges of ALPHA + (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), period (%2E), + underscore (%5F), or tilde (%7E) should not be created by URI + producers and, when found in a URI, should be decoded to their + corresponding unreserved characters by URI normalizers. + +2.4. When to Encode or Decode + + Under normal circumstances, the only time when octets within a URI + are percent-encoded is during the process of producing the URI from + its component parts. This is when an implementation determines which + of the reserved characters are to be used as subcomponent delimiters + and which can be safely used as data. Once produced, a URI is always + in its percent-encoded form. + + When a URI is dereferenced, the components and subcomponents + significant to the scheme-specific dereferencing process (if any) + must be parsed and separated before the percent-encoded octets within + those components can be safely decoded, as otherwise the data may be + mistaken for component delimiters. The only exception is for + percent-encoded octets corresponding to characters in the unreserved + set, which can be decoded at any time. For example, the octet + corresponding to the tilde ("~") character is often encoded as "%7E" + by older URI processing implementations; the "%7E" can be replaced by + "~" without changing its interpretation. + + Because the percent ("%") character serves as the indicator for + percent-encoded octets, it must be percent-encoded as "%25" for that + octet to be used as data within a URI. Implementations must not + percent-encode or decode the same string more than once, as decoding + an already decoded string might lead to misinterpreting a percent + data octet as the beginning of a percent-encoding, or vice versa in + the case of percent-encoding an already percent-encoded string. + +2.5. Identifying Data + + URI characters provide identifying data for each of the URI + components, serving as an external interface for identification + between systems. Although the presence and nature of the URI + production interface is hidden from clients that use its URIs (and is + thus beyond the scope of the interoperability requirements defined by + this specification), it is a frequent source of confusion and errors + in the interpretation of URI character issues. Implementers have to + be aware that there are multiple character encodings involved in the + + + +Berners-Lee, et al. Standards Track [Page 14] + +RFC 3986 URI Generic Syntax January 2005 + + + production and transmission of URIs: local name and data encoding, + public interface encoding, URI character encoding, data format + encoding, and protocol encoding. + + Local names, such as file system names, are stored with a local + character encoding. URI producing applications (e.g., origin + servers) will typically use the local encoding as the basis for + producing meaningful names. The URI producer will transform the + local encoding to one that is suitable for a public interface and + then transform the public interface encoding into the restricted set + of URI characters (reserved, unreserved, and percent-encodings). + Those characters are, in turn, encoded as octets to be used as a + reference within a data format (e.g., a document charset), and such + data formats are often subsequently encoded for transmission over + Internet protocols. + + For most systems, an unreserved character appearing within a URI + component is interpreted as representing the data octet corresponding + to that character's encoding in US-ASCII. Consumers of URIs assume + that the letter "X" corresponds to the octet "01011000", and even + when that assumption is incorrect, there is no harm in making it. A + system that internally provides identifiers in the form of a + different character encoding, such as EBCDIC, will generally perform + character translation of textual identifiers to UTF-8 [STD63] (or + some other superset of the US-ASCII character encoding) at an + internal interface, thereby providing more meaningful identifiers + than those resulting from simply percent-encoding the original + octets. + + For example, consider an information service that provides data, + stored locally using an EBCDIC-based file system, to clients on the + Internet through an HTTP server. When an author creates a file with + the name "Laguna Beach" on that file system, the "http" URI + corresponding to that resource is expected to contain the meaningful + string "Laguna%20Beach". If, however, that server produces URIs by + using an overly simplistic raw octet mapping, then the result would + be a URI containing "%D3%81%87%A4%95%81@%C2%85%81%83%88". An + internal transcoding interface fixes this problem by transcoding the + local name to a superset of US-ASCII prior to producing the URI. + Naturally, proper interpretation of an incoming URI on such an + interface requires that percent-encoded octets be decoded (e.g., + "%20" to SP) before the reverse transcoding is applied to obtain the + local name. + + In some cases, the internal interface between a URI component and the + identifying data that it has been crafted to represent is much less + direct than a character encoding translation. For example, portions + of a URI might reflect a query on non-ASCII data, or numeric + + + +Berners-Lee, et al. Standards Track [Page 15] + +RFC 3986 URI Generic Syntax January 2005 + + + coordinates on a map. Likewise, a URI scheme may define components + with additional encoding requirements that are applied prior to + forming the component and producing the URI. + + When a new URI scheme defines a component that represents textual + data consisting of characters from the Universal Character Set [UCS], + the data should first be encoded as octets according to the UTF-8 + character encoding [STD63]; then only those octets that do not + correspond to characters in the unreserved set should be percent- + encoded. For example, the character A would be represented as "A", + the character LATIN CAPITAL LETTER A WITH GRAVE would be represented + as "%C3%80", and the character KATAKANA LETTER A would be represented + as "%E3%82%A2". + +3. Syntax Components + + The generic URI syntax consists of a hierarchical sequence of + components referred to as the scheme, authority, path, query, and + fragment. + + URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] + + hier-part = "//" authority path-abempty + / path-absolute + / path-rootless + / path-empty + + The scheme and path components are required, though the path may be + empty (no characters). When authority is present, the path must + either be empty or begin with a slash ("/") character. When + authority is not present, the path cannot begin with two slash + characters ("//"). These restrictions result in five different ABNF + rules for a path (Section 3.3), only one of which will match any + given URI reference. + + The following are two example URIs and their component parts: + + foo://example.com:8042/over/there?name=ferret#nose + \_/ \______________/\_________/ \_________/ \__/ + | | | | | + scheme authority path query fragment + | _____________________|__ + / \ / \ + urn:example:animal:ferret:nose + + + + + + + +Berners-Lee, et al. Standards Track [Page 16] + +RFC 3986 URI Generic Syntax January 2005 + + +3.1. Scheme + + Each URI begins with a scheme name that refers to a specification for + assigning identifiers within that scheme. As such, the URI syntax is + a federated and extensible naming system wherein each scheme's + specification may further restrict the syntax and semantics of + identifiers using that scheme. + + Scheme names consist of a sequence of characters beginning with a + letter and followed by any combination of letters, digits, plus + ("+"), period ("."), or hyphen ("-"). Although schemes are case- + insensitive, the canonical form is lowercase and documents that + specify schemes must do so with lowercase letters. An implementation + should accept uppercase letters as equivalent to lowercase in scheme + names (e.g., allow "HTTP" as well as "http") for the sake of + robustness but should only produce lowercase scheme names for + consistency. + + scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + + Individual schemes are not specified by this document. The process + for registration of new URI schemes is defined separately by [BCP35]. + The scheme registry maintains the mapping between scheme names and + their specifications. Advice for designers of new URI schemes can be + found in [RFC2718]. URI scheme specifications must define their own + syntax so that all strings matching their scheme-specific syntax will + also match the <absolute-URI> grammar, as described in Section 4.3. + + When presented with a URI that violates one or more scheme-specific + restrictions, the scheme-specific resolution process should flag the + reference as an error rather than ignore the unused parts; doing so + reduces the number of equivalent URIs and helps detect abuses of the + generic syntax, which might indicate that the URI has been + constructed to mislead the user (Section 7.6). + +3.2. Authority + + Many URI schemes include a hierarchical element for a naming + authority so that governance of the name space defined by the + remainder of the URI is delegated to that authority (which may, in + turn, delegate it further). The generic syntax provides a common + means for distinguishing an authority based on a registered name or + server address, along with optional port and user information. + + The authority component is preceded by a double slash ("//") and is + terminated by the next slash ("/"), question mark ("?"), or number + sign ("#") character, or by the end of the URI. + + + + +Berners-Lee, et al. Standards Track [Page 17] + +RFC 3986 URI Generic Syntax January 2005 + + + authority = [ userinfo "@" ] host [ ":" port ] + + URI producers and normalizers should omit the ":" delimiter that + separates host from port if the port component is empty. Some + schemes do not allow the userinfo and/or port subcomponents. + + If a URI contains an authority component, then the path component + must either be empty or begin with a slash ("/") character. Non- + validating parsers (those that merely separate a URI reference into + its major components) will often ignore the subcomponent structure of + authority, treating it as an opaque string from the double-slash to + the first terminating delimiter, until such time as the URI is + dereferenced. + +3.2.1. User Information + + The userinfo subcomponent may consist of a user name and, optionally, + scheme-specific information about how to gain authorization to access + the resource. The user information, if present, is followed by a + commercial at-sign ("@") that delimits it from the host. + + userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + + Use of the format "user:password" in the userinfo field is + deprecated. Applications should not render as clear text any data + after the first colon (":") character found within a userinfo + subcomponent unless the data after the colon is the empty string + (indicating no password). Applications may choose to ignore or + reject such data when it is received as part of a reference and + should reject the storage of such data in unencrypted form. The + passing of authentication information in clear text has proven to be + a security risk in almost every case where it has been used. + + Applications that render a URI for the sake of user feedback, such as + in graphical hypertext browsing, should render userinfo in a way that + is distinguished from the rest of a URI, when feasible. Such + rendering will assist the user in cases where the userinfo has been + misleadingly crafted to look like a trusted domain name + (Section 7.6). + +3.2.2. Host + + The host subcomponent of authority is identified by an IP literal + encapsulated within square brackets, an IPv4 address in dotted- + decimal form, or a registered name. The host subcomponent is case- + insensitive. The presence of a host subcomponent within a URI does + not imply that the scheme requires access to the given host on the + Internet. In many cases, the host syntax is used only for the sake + + + +Berners-Lee, et al. Standards Track [Page 18] + +RFC 3986 URI Generic Syntax January 2005 + + + of reusing the existing registration process created and deployed for + DNS, thus obtaining a globally unique name without the cost of + deploying another registry. However, such use comes with its own + costs: domain name ownership may change over time for reasons not + anticipated by the URI producer. In other cases, the data within the + host component identifies a registered name that has nothing to do + with an Internet host. We use the name "host" for the ABNF rule + because that is its most common purpose, not its only purpose. + + host = IP-literal / IPv4address / reg-name + + The syntax rule for host is ambiguous because it does not completely + distinguish between an IPv4address and a reg-name. In order to + disambiguate the syntax, we apply the "first-match-wins" algorithm: + If host matches the rule for IPv4address, then it should be + considered an IPv4 address literal and not a reg-name. Although host + is case-insensitive, producers and normalizers should use lowercase + for registered names and hexadecimal addresses for the sake of + uniformity, while only using uppercase letters for percent-encodings. + + A host identified by an Internet Protocol literal address, version 6 + [RFC3513] or later, is distinguished by enclosing the IP literal + within square brackets ("[" and "]"). This is the only place where + square bracket characters are allowed in the URI syntax. In + anticipation of future, as-yet-undefined IP literal address formats, + an implementation may use an optional version flag to indicate such a + format explicitly rather than rely on heuristic determination. + + IP-literal = "[" ( IPv6address / IPvFuture ) "]" + + IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) + + The version flag does not indicate the IP version; rather, it + indicates future versions of the literal format. As such, + implementations must not provide the version flag for the existing + IPv4 and IPv6 literal address forms described below. If a URI + containing an IP-literal that starts with "v" (case-insensitive), + indicating that the version flag is present, is dereferenced by an + application that does not know the meaning of that version flag, then + the application should return an appropriate error for "address + mechanism not supported". + + A host identified by an IPv6 literal address is represented inside + the square brackets without a preceding version flag. The ABNF + provided here is a translation of the text definition of an IPv6 + literal address provided in [RFC3513]. This syntax does not support + IPv6 scoped addressing zone identifiers. + + + + +Berners-Lee, et al. Standards Track [Page 19] + +RFC 3986 URI Generic Syntax January 2005 + + + A 128-bit IPv6 address is divided into eight 16-bit pieces. Each + piece is represented numerically in case-insensitive hexadecimal, + using one to four hexadecimal digits (leading zeroes are permitted). + The eight encoded pieces are given most-significant first, separated + by colon characters. Optionally, the least-significant two pieces + may instead be represented in IPv4 address textual format. A + sequence of one or more consecutive zero-valued 16-bit pieces within + the address may be elided, omitting all their digits and leaving + exactly two consecutive colons in their place to mark the elision. + + IPv6address = 6( h16 ":" ) ls32 + / "::" 5( h16 ":" ) ls32 + / [ h16 ] "::" 4( h16 ":" ) ls32 + / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + / [ *4( h16 ":" ) h16 ] "::" ls32 + / [ *5( h16 ":" ) h16 ] "::" h16 + / [ *6( h16 ":" ) h16 ] "::" + + ls32 = ( h16 ":" h16 ) / IPv4address + ; least-significant 32 bits of address + + h16 = 1*4HEXDIG + ; 16 bits of address represented in hexadecimal + + A host identified by an IPv4 literal address is represented in + dotted-decimal notation (a sequence of four decimal numbers in the + range 0 to 255, separated by "."), as described in [RFC1123] by + reference to [RFC0952]. Note that other forms of dotted notation may + be interpreted on some platforms, as described in Section 7.4, but + only the dotted-decimal form of four octets is allowed by this + grammar. + + IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + + dec-octet = DIGIT ; 0-9 + / %x31-39 DIGIT ; 10-99 + / "1" 2DIGIT ; 100-199 + / "2" %x30-34 DIGIT ; 200-249 + / "25" %x30-35 ; 250-255 + + A host identified by a registered name is a sequence of characters + usually intended for lookup within a locally defined host or service + name registry, though the URI's scheme-specific semantics may require + that a specific registry (or fixed name table) be used instead. The + most common name registry mechanism is the Domain Name System (DNS). + A registered name intended for lookup in the DNS uses the syntax + + + +Berners-Lee, et al. Standards Track [Page 20] + +RFC 3986 URI Generic Syntax January 2005 + + + defined in Section 3.5 of [RFC1034] and Section 2.1 of [RFC1123]. + Such a name consists of a sequence of domain labels separated by ".", + each domain label starting and ending with an alphanumeric character + and possibly also containing "-" characters. The rightmost domain + label of a fully qualified domain name in DNS may be followed by a + single "." and should be if it is necessary to distinguish between + the complete domain name and some local domain. + + reg-name = *( unreserved / pct-encoded / sub-delims ) + + If the URI scheme defines a default for host, then that default + applies when the host subcomponent is undefined or when the + registered name is empty (zero length). For example, the "file" URI + scheme is defined so that no authority, an empty host, and + "localhost" all mean the end-user's machine, whereas the "http" + scheme considers a missing authority or empty host invalid. + + This specification does not mandate a particular registered name + lookup technology and therefore does not restrict the syntax of reg- + name beyond what is necessary for interoperability. Instead, it + delegates the issue of registered name syntax conformance to the + operating system of each application performing URI resolution, and + that operating system decides what it will allow for the purpose of + host identification. A URI resolution implementation might use DNS, + host tables, yellow pages, NetInfo, WINS, or any other system for + lookup of registered names. However, a globally scoped naming + system, such as DNS fully qualified domain names, is necessary for + URIs intended to have global scope. URI producers should use names + that conform to the DNS syntax, even when use of DNS is not + immediately apparent, and should limit these names to no more than + 255 characters in length. + + The reg-name syntax allows percent-encoded octets in order to + represent non-ASCII registered names in a uniform way that is + independent of the underlying name resolution technology. Non-ASCII + characters must first be encoded according to UTF-8 [STD63], and then + each octet of the corresponding UTF-8 sequence must be percent- + encoded to be represented as URI characters. URI producing + applications must not use percent-encoding in host unless it is used + to represent a UTF-8 character sequence. When a non-ASCII registered + name represents an internationalized domain name intended for + resolution via the DNS, the name must be transformed to the IDNA + encoding [RFC3490] prior to name lookup. URI producers should + provide these registered names in the IDNA encoding, rather than a + percent-encoding, if they wish to maximize interoperability with + legacy URI resolvers. + + + + + +Berners-Lee, et al. Standards Track [Page 21] + +RFC 3986 URI Generic Syntax January 2005 + + +3.2.3. Port + + The port subcomponent of authority is designated by an optional port + number in decimal following the host and delimited from it by a + single colon (":") character. + + port = *DIGIT + + A scheme may define a default port. For example, the "http" scheme + defines a default port of "80", corresponding to its reserved TCP + port number. The type of port designated by the port number (e.g., + TCP, UDP, SCTP) is defined by the URI scheme. URI producers and + normalizers should omit the port component and its ":" delimiter if + port is empty or if its value would be the same as that of the + scheme's default. + +3.3. Path + + The path component contains data, usually organized in hierarchical + form, that, along with data in the non-hierarchical query component + (Section 3.4), serves to identify a resource within the scope of the + URI's scheme and naming authority (if any). The path is terminated + by the first question mark ("?") or number sign ("#") character, or + by the end of the URI. + + If a URI contains an authority component, then the path component + must either be empty or begin with a slash ("/") character. If a URI + does not contain an authority component, then the path cannot begin + with two slash characters ("//"). In addition, a URI reference + (Section 4.1) may be a relative-path reference, in which case the + first path segment cannot contain a colon (":") character. The ABNF + requires five separate rules to disambiguate these cases, only one of + which will match the path substring within a given URI reference. We + use the generic term "path component" to describe the URI substring + matched by the parser to one of these rules. + + path = path-abempty ; begins with "/" or is empty + / path-absolute ; begins with "/" but not "//" + / path-noscheme ; begins with a non-colon segment + / path-rootless ; begins with a segment + / path-empty ; zero characters + + path-abempty = *( "/" segment ) + path-absolute = "/" [ segment-nz *( "/" segment ) ] + path-noscheme = segment-nz-nc *( "/" segment ) + path-rootless = segment-nz *( "/" segment ) + path-empty = 0<pchar> + + + + +Berners-Lee, et al. Standards Track [Page 22] + +RFC 3986 URI Generic Syntax January 2005 + + + segment = *pchar + segment-nz = 1*pchar + segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + ; non-zero-length segment without any colon ":" + + pchar = unreserved / pct-encoded / sub-delims / ":" / "@" + + A path consists of a sequence of path segments separated by a slash + ("/") character. A path is always defined for a URI, though the + defined path may be empty (zero length). Use of the slash character + to indicate hierarchy is only required when a URI will be used as the + context for relative references. For example, the URI + <mailto:[email protected]> has a path of "[email protected]", whereas + the URI <foo://info.example.com?fred> has an empty path. + + The path segments "." and "..", also known as dot-segments, are + defined for relative reference within the path name hierarchy. They + are intended for use at the beginning of a relative-path reference + (Section 4.2) to indicate relative position within the hierarchical + tree of names. This is similar to their role within some operating + systems' file directory structures to indicate the current directory + and parent directory, respectively. However, unlike in a file + system, these dot-segments are only interpreted within the URI path + hierarchy and are removed as part of the resolution process (Section + 5.2). + + Aside from dot-segments in hierarchical paths, a path segment is + considered opaque by the generic syntax. URI producing applications + often use the reserved characters allowed in a segment to delimit + scheme-specific or dereference-handler-specific subcomponents. For + example, the semicolon (";") and equals ("=") reserved characters are + often used to delimit parameters and parameter values applicable to + that segment. The comma (",") reserved character is often used for + similar purposes. For example, one URI producer might use a segment + such as "name;v=1.1" to indicate a reference to version 1.1 of + "name", whereas another might use a segment such as "name,1.1" to + indicate the same. Parameter types may be defined by scheme-specific + semantics, but in most cases the syntax of a parameter is specific to + the implementation of the URI's dereferencing algorithm. + +3.4. Query + + The query component contains non-hierarchical data that, along with + data in the path component (Section 3.3), serves to identify a + resource within the scope of the URI's scheme and naming authority + (if any). The query component is indicated by the first question + mark ("?") character and terminated by a number sign ("#") character + or by the end of the URI. + + + +Berners-Lee, et al. Standards Track [Page 23] + +RFC 3986 URI Generic Syntax January 2005 + + + query = *( pchar / "/" / "?" ) + + The characters slash ("/") and question mark ("?") may represent data + within the query component. Beware that some older, erroneous + implementations may not handle such data correctly when it is used as + the base URI for relative references (Section 5.1), apparently + because they fail to distinguish query data from path data when + looking for hierarchical separators. However, as query components + are often used to carry identifying information in the form of + "key=value" pairs and one frequently used value is a reference to + another URI, it is sometimes better for usability to avoid percent- + encoding those characters. + +3.5. Fragment + + The fragment identifier component of a URI allows indirect + identification of a secondary resource by reference to a primary + resource and additional identifying information. The identified + secondary resource may be some portion or subset of the primary + resource, some view on representations of the primary resource, or + some other resource defined or described by those representations. A + fragment identifier component is indicated by the presence of a + number sign ("#") character and terminated by the end of the URI. + + fragment = *( pchar / "/" / "?" ) + + The semantics of a fragment identifier are defined by the set of + representations that might result from a retrieval action on the + primary resource. The fragment's format and resolution is therefore + dependent on the media type [RFC2046] of a potentially retrieved + representation, even though such a retrieval is only performed if the + URI is dereferenced. If no such representation exists, then the + semantics of the fragment are considered unknown and are effectively + unconstrained. Fragment identifier semantics are independent of the + URI scheme and thus cannot be redefined by scheme specifications. + + Individual media types may define their own restrictions on or + structures within the fragment identifier syntax for specifying + different types of subsets, views, or external references that are + identifiable as secondary resources by that media type. If the + primary resource has multiple representations, as is often the case + for resources whose representation is selected based on attributes of + the retrieval request (a.k.a., content negotiation), then whatever is + identified by the fragment should be consistent across all of those + representations. Each representation should either define the + fragment so that it corresponds to the same secondary resource, + regardless of how it is represented, or should leave the fragment + undefined (i.e., not found). + + + +Berners-Lee, et al. Standards Track [Page 24] + +RFC 3986 URI Generic Syntax January 2005 + + + As with any URI, use of a fragment identifier component does not + imply that a retrieval action will take place. A URI with a fragment + identifier may be used to refer to the secondary resource without any + implication that the primary resource is accessible or will ever be + accessed. + + Fragment identifiers have a special role in information retrieval + systems as the primary form of client-side indirect referencing, + allowing an author to specifically identify aspects of an existing + resource that are only indirectly provided by the resource owner. As + such, the fragment identifier is not used in the scheme-specific + processing of a URI; instead, the fragment identifier is separated + from the rest of the URI prior to a dereference, and thus the + identifying information within the fragment itself is dereferenced + solely by the user agent, regardless of the URI scheme. Although + this separate handling is often perceived to be a loss of + information, particularly for accurate redirection of references as + resources move over time, it also serves to prevent information + providers from denying reference authors the right to refer to + information within a resource selectively. Indirect referencing also + provides additional flexibility and extensibility to systems that use + URIs, as new media types are easier to define and deploy than new + schemes of identification. + + The characters slash ("/") and question mark ("?") are allowed to + represent data within the fragment identifier. Beware that some + older, erroneous implementations may not handle this data correctly + when it is used as the base URI for relative references (Section + 5.1). + +4. Usage + + When applications make reference to a URI, they do not always use the + full form of reference defined by the "URI" syntax rule. To save + space and take advantage of hierarchical locality, many Internet + protocol elements and media type formats allow an abbreviation of a + URI, whereas others restrict the syntax to a particular form of URI. + We define the most common forms of reference syntax in this + specification because they impact and depend upon the design of the + generic syntax, requiring a uniform parsing algorithm in order to be + interpreted consistently. + +4.1. URI Reference + + URI-reference is used to denote the most common usage of a resource + identifier. + + URI-reference = URI / relative-ref + + + +Berners-Lee, et al. Standards Track [Page 25] + +RFC 3986 URI Generic Syntax January 2005 + + + A URI-reference is either a URI or a relative reference. If the + URI-reference's prefix does not match the syntax of a scheme followed + by its colon separator, then the URI-reference is a relative + reference. + + A URI-reference is typically parsed first into the five URI + components, in order to determine what components are present and + whether the reference is relative. Then, each component is parsed + for its subparts and their validation. The ABNF of URI-reference, + along with the "first-match-wins" disambiguation rule, is sufficient + to define a validating parser for the generic syntax. Readers + familiar with regular expressions should see Appendix B for an + example of a non-validating URI-reference parser that will take any + given string and extract the URI components. + +4.2. Relative Reference + + A relative reference takes advantage of the hierarchical syntax + (Section 1.2.3) to express a URI reference relative to the name space + of another hierarchical URI. + + relative-ref = relative-part [ "?" query ] [ "#" fragment ] + + relative-part = "//" authority path-abempty + / path-absolute + / path-noscheme + / path-empty + + The URI referred to by a relative reference, also known as the target + URI, is obtained by applying the reference resolution algorithm of + Section 5. + + A relative reference that begins with two slash characters is termed + a network-path reference; such references are rarely used. A + relative reference that begins with a single slash character is + termed an absolute-path reference. A relative reference that does + not begin with a slash character is termed a relative-path reference. + + A path segment that contains a colon character (e.g., "this:that") + cannot be used as the first segment of a relative-path reference, as + it would be mistaken for a scheme name. Such a segment must be + preceded by a dot-segment (e.g., "./this:that") to make a relative- + path reference. + + + + + + + + +Berners-Lee, et al. Standards Track [Page 26] + +RFC 3986 URI Generic Syntax January 2005 + + +4.3. Absolute URI + + Some protocol elements allow only the absolute form of a URI without + a fragment identifier. For example, defining a base URI for later + use by relative references calls for an absolute-URI syntax rule that + does not allow a fragment. + + absolute-URI = scheme ":" hier-part [ "?" query ] + + URI scheme specifications must define their own syntax so that all + strings matching their scheme-specific syntax will also match the + <absolute-URI> grammar. Scheme specifications will not define + fragment identifier syntax or usage, regardless of its applicability + to resources identifiable via that scheme, as fragment identification + is orthogonal to scheme definition. However, scheme specifications + are encouraged to include a wide range of examples, including + examples that show use of the scheme's URIs with fragment identifiers + when such usage is appropriate. + +4.4. Same-Document Reference + + When a URI reference refers to a URI that is, aside from its fragment + component (if any), identical to the base URI (Section 5.1), that + reference is called a "same-document" reference. The most frequent + examples of same-document references are relative references that are + empty or include only the number sign ("#") separator followed by a + fragment identifier. + + When a same-document reference is dereferenced for a retrieval + action, the target of that reference is defined to be within the same + entity (representation, document, or message) as the reference; + therefore, a dereference should not result in a new retrieval action. + + Normalization of the base and target URIs prior to their comparison, + as described in Sections 6.2.2 and 6.2.3, is allowed but rarely + performed in practice. Normalization may increase the set of same- + document references, which may be of benefit to some caching + applications. As such, reference authors should not assume that a + slightly different, though equivalent, reference URI will (or will + not) be interpreted as a same-document reference by any given + application. + +4.5. Suffix Reference + + The URI syntax is designed for unambiguous reference to resources and + extensibility via the URI scheme. However, as URI identification and + usage have become commonplace, traditional media (television, radio, + newspapers, billboards, etc.) have increasingly used a suffix of the + + + +Berners-Lee, et al. Standards Track [Page 27] + +RFC 3986 URI Generic Syntax January 2005 + + + URI as a reference, consisting of only the authority and path + portions of the URI, such as + + www.w3.org/Addressing/ + + or simply a DNS registered name on its own. Such references are + primarily intended for human interpretation rather than for machines, + with the assumption that context-based heuristics are sufficient to + complete the URI (e.g., most registered names beginning with "www" + are likely to have a URI prefix of "http://"). Although there is no + standard set of heuristics for disambiguating a URI suffix, many + client implementations allow them to be entered by the user and + heuristically resolved. + + Although this practice of using suffix references is common, it + should be avoided whenever possible and should never be used in + situations where long-term references are expected. The heuristics + noted above will change over time, particularly when a new URI scheme + becomes popular, and are often incorrect when used out of context. + Furthermore, they can lead to security issues along the lines of + those described in [RFC1535]. + + As a URI suffix has the same syntax as a relative-path reference, a + suffix reference cannot be used in contexts where a relative + reference is expected. As a result, suffix references are limited to + places where there is no defined base URI, such as dialog boxes and + off-line advertisements. + +5. Reference Resolution + + This section defines the process of resolving a URI reference within + a context that allows relative references so that the result is a + string matching the <URI> syntax rule of Section 3. + +5.1. Establishing a Base URI + + The term "relative" implies that a "base URI" exists against which + the relative reference is applied. Aside from fragment-only + references (Section 4.4), relative references are only usable when a + base URI is known. A base URI must be established by the parser + prior to parsing URI references that might be relative. A base URI + must conform to the <absolute-URI> syntax rule (Section 4.3). If the + base URI is obtained from a URI reference, then that reference must + be converted to absolute form and stripped of any fragment component + prior to its use as a base URI. + + + + + + +Berners-Lee, et al. Standards Track [Page 28] + +RFC 3986 URI Generic Syntax January 2005 + + + The base URI of a reference can be established in one of four ways, + discussed below in order of precedence. The order of precedence can + be thought of in terms of layers, where the innermost defined base + URI has the highest precedence. This can be visualized graphically + as follows: + + .----------------------------------------------------------. + | .----------------------------------------------------. | + | | .----------------------------------------------. | | + | | | .----------------------------------------. | | | + | | | | .----------------------------------. | | | | + | | | | | <relative-reference> | | | | | + | | | | `----------------------------------' | | | | + | | | | (5.1.1) Base URI embedded in content | | | | + | | | `----------------------------------------' | | | + | | | (5.1.2) Base URI of the encapsulating entity | | | + | | | (message, representation, or none) | | | + | | `----------------------------------------------' | | + | | (5.1.3) URI used to retrieve the entity | | + | `----------------------------------------------------' | + | (5.1.4) Default Base URI (application-dependent) | + `----------------------------------------------------------' + +5.1.1. Base URI Embedded in Content + + Within certain media types, a base URI for relative references can be + embedded within the content itself so that it can be readily obtained + by a parser. This can be useful for descriptive documents, such as + tables of contents, which may be transmitted to others through + protocols other than their usual retrieval context (e.g., email or + USENET news). + + It is beyond the scope of this specification to specify how, for each + media type, a base URI can be embedded. The appropriate syntax, when + available, is described by the data format specification associated + with each media type. + +5.1.2. Base URI from the Encapsulating Entity + + If no base URI is embedded, the base URI is defined by the + representation's retrieval context. For a document that is enclosed + within another entity, such as a message or archive, the retrieval + context is that entity. Thus, the default base URI of a + representation is the base URI of the entity in which the + representation is encapsulated. + + + + + + +Berners-Lee, et al. Standards Track [Page 29] + +RFC 3986 URI Generic Syntax January 2005 + + + A mechanism for embedding a base URI within MIME container types + (e.g., the message and multipart types) is defined by MHTML + [RFC2557]. Protocols that do not use the MIME message header syntax, + but that do allow some form of tagged metadata to be included within + messages, may define their own syntax for defining a base URI as part + of a message. + +5.1.3. Base URI from the Retrieval URI + + If no base URI is embedded and the representation is not encapsulated + within some other entity, then, if a URI was used to retrieve the + representation, that URI shall be considered the base URI. Note that + if the retrieval was the result of a redirected request, the last URI + used (i.e., the URI that resulted in the actual retrieval of the + representation) is the base URI. + +5.1.4. Default Base URI + + If none of the conditions described above apply, then the base URI is + defined by the context of the application. As this definition is + necessarily application-dependent, failing to define a base URI by + using one of the other methods may result in the same content being + interpreted differently by different types of applications. + + A sender of a representation containing relative references is + responsible for ensuring that a base URI for those references can be + established. Aside from fragment-only references, relative + references can only be used reliably in situations where the base URI + is well defined. + +5.2. Relative Resolution + + This section describes an algorithm for converting a URI reference + that might be relative to a given base URI into the parsed components + of the reference's target. The components can then be recomposed, as + described in Section 5.3, to form the target URI. This algorithm + provides definitive results that can be used to test the output of + other implementations. Applications may implement relative reference + resolution by using some other algorithm, provided that the results + match what would be given by this one. + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 30] + +RFC 3986 URI Generic Syntax January 2005 + + +5.2.1. Pre-parse the Base URI + + The base URI (Base) is established according to the procedure of + Section 5.1 and parsed into the five main components described in + Section 3. Note that only the scheme component is required to be + present in a base URI; the other components may be empty or + undefined. A component is undefined if its associated delimiter does + not appear in the URI reference; the path component is never + undefined, though it may be empty. + + Normalization of the base URI, as described in Sections 6.2.2 and + 6.2.3, is optional. A URI reference must be transformed to its + target URI before it can be normalized. + +5.2.2. Transform References + + For each URI reference (R), the following pseudocode describes an + algorithm for transforming R into its target URI (T): + + -- The URI reference is parsed into the five URI components + -- + (R.scheme, R.authority, R.path, R.query, R.fragment) = parse(R); + + -- A non-strict parser may ignore a scheme in the reference + -- if it is identical to the base URI's scheme. + -- + if ((not strict) and (R.scheme == Base.scheme)) then + undefine(R.scheme); + endif; + + + + + + + + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 31] + +RFC 3986 URI Generic Syntax January 2005 + + + if defined(R.scheme) then + T.scheme = R.scheme; + T.authority = R.authority; + T.path = remove_dot_segments(R.path); + T.query = R.query; + else + if defined(R.authority) then + T.authority = R.authority; + T.path = remove_dot_segments(R.path); + T.query = R.query; + else + if (R.path == "") then + T.path = Base.path; + if defined(R.query) then + T.query = R.query; + else + T.query = Base.query; + endif; + else + if (R.path starts-with "/") then + T.path = remove_dot_segments(R.path); + else + T.path = merge(Base.path, R.path); + T.path = remove_dot_segments(T.path); + endif; + T.query = R.query; + endif; + T.authority = Base.authority; + endif; + T.scheme = Base.scheme; + endif; + + T.fragment = R.fragment; + +5.2.3. Merge Paths + + The pseudocode above refers to a "merge" routine for merging a + relative-path reference with the path of the base URI. This is + accomplished as follows: + + o If the base URI has a defined authority component and an empty + path, then return a string consisting of "/" concatenated with the + reference's path; otherwise, + + + + + + + + +Berners-Lee, et al. Standards Track [Page 32] + +RFC 3986 URI Generic Syntax January 2005 + + + o return a string consisting of the reference's path component + appended to all but the last segment of the base URI's path (i.e., + excluding any characters after the right-most "/" in the base URI + path, or excluding the entire base URI path if it does not contain + any "/" characters). + +5.2.4. Remove Dot Segments + + The pseudocode also refers to a "remove_dot_segments" routine for + interpreting and removing the special "." and ".." complete path + segments from a referenced path. This is done after the path is + extracted from a reference, whether or not the path was relative, in + order to remove any invalid or extraneous dot-segments prior to + forming the target URI. Although there are many ways to accomplish + this removal process, we describe a simple method using two string + buffers. + + 1. The input buffer is initialized with the now-appended path + components and the output buffer is initialized to the empty + string. + + 2. While the input buffer is not empty, loop as follows: + + A. If the input buffer begins with a prefix of "../" or "./", + then remove that prefix from the input buffer; otherwise, + + B. if the input buffer begins with a prefix of "/./" or "/.", + where "." is a complete path segment, then replace that + prefix with "/" in the input buffer; otherwise, + + C. if the input buffer begins with a prefix of "/../" or "/..", + where ".." is a complete path segment, then replace that + prefix with "/" in the input buffer and remove the last + segment and its preceding "/" (if any) from the output + buffer; otherwise, + + D. if the input buffer consists only of "." or "..", then remove + that from the input buffer; otherwise, + + E. move the first path segment in the input buffer to the end of + the output buffer, including the initial "/" character (if + any) and any subsequent characters up to, but not including, + the next "/" character or the end of the input buffer. + + 3. Finally, the output buffer is returned as the result of + remove_dot_segments. + + + + + +Berners-Lee, et al. Standards Track [Page 33] + +RFC 3986 URI Generic Syntax January 2005 + + + Note that dot-segments are intended for use in URI references to + express an identifier relative to the hierarchy of names in the base + URI. The remove_dot_segments algorithm respects that hierarchy by + removing extra dot-segments rather than treat them as an error or + leaving them to be misinterpreted by dereference implementations. + + The following illustrates how the above steps are applied for two + examples of merged paths, showing the state of the two buffers after + each step. + + STEP OUTPUT BUFFER INPUT BUFFER + + 1 : /a/b/c/./../../g + 2E: /a /b/c/./../../g + 2E: /a/b /c/./../../g + 2E: /a/b/c /./../../g + 2B: /a/b/c /../../g + 2C: /a/b /../g + 2C: /a /g + 2E: /a/g + + STEP OUTPUT BUFFER INPUT BUFFER + + 1 : mid/content=5/../6 + 2E: mid /content=5/../6 + 2E: mid/content=5 /../6 + 2C: mid /6 + 2E: mid/6 + + Some applications may find it more efficient to implement the + remove_dot_segments algorithm by using two segment stacks rather than + strings. + + Note: Beware that some older, erroneous implementations will fail + to separate a reference's query component from its path component + prior to merging the base and reference paths, resulting in an + interoperability failure if the query component contains the + strings "/../" or "/./". + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 34] + +RFC 3986 URI Generic Syntax January 2005 + + +5.3. Component Recomposition + + Parsed URI components can be recomposed to obtain the corresponding + URI reference string. Using pseudocode, this would be: + + result = "" + + if defined(scheme) then + append scheme to result; + append ":" to result; + endif; + + if defined(authority) then + append "//" to result; + append authority to result; + endif; + + append path to result; + + if defined(query) then + append "?" to result; + append query to result; + endif; + + if defined(fragment) then + append "#" to result; + append fragment to result; + endif; + + return result; + + Note that we are careful to preserve the distinction between a + component that is undefined, meaning that its separator was not + present in the reference, and a component that is empty, meaning that + the separator was present and was immediately followed by the next + component separator or the end of the reference. + +5.4. Reference Resolution Examples + + Within a representation with a well defined base URI of + + http://a/b/c/d;p?q + + a relative reference is transformed to its target URI as follows. + + + + + + + +Berners-Lee, et al. Standards Track [Page 35] + +RFC 3986 URI Generic Syntax January 2005 + + +5.4.1. Normal Examples + + "g:h" = "g:h" + "g" = "http://a/b/c/g" + "./g" = "http://a/b/c/g" + "g/" = "http://a/b/c/g/" + "/g" = "http://a/g" + "//g" = "http://g" + "?y" = "http://a/b/c/d;p?y" + "g?y" = "http://a/b/c/g?y" + "#s" = "http://a/b/c/d;p?q#s" + "g#s" = "http://a/b/c/g#s" + "g?y#s" = "http://a/b/c/g?y#s" + ";x" = "http://a/b/c/;x" + "g;x" = "http://a/b/c/g;x" + "g;x?y#s" = "http://a/b/c/g;x?y#s" + "" = "http://a/b/c/d;p?q" + "." = "http://a/b/c/" + "./" = "http://a/b/c/" + ".." = "http://a/b/" + "../" = "http://a/b/" + "../g" = "http://a/b/g" + "../.." = "http://a/" + "../../" = "http://a/" + "../../g" = "http://a/g" + +5.4.2. Abnormal Examples + + Although the following abnormal examples are unlikely to occur in + normal practice, all URI parsers should be capable of resolving them + consistently. Each example uses the same base as that above. + + Parsers must be careful in handling cases where there are more ".." + segments in a relative-path reference than there are hierarchical + levels in the base URI's path. Note that the ".." syntax cannot be + used to change the authority component of a URI. + + "../../../g" = "http://a/g" + "../../../../g" = "http://a/g" + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 36] + +RFC 3986 URI Generic Syntax January 2005 + + + Similarly, parsers must remove the dot-segments "." and ".." when + they are complete components of a path, but not when they are only + part of a segment. + + "/./g" = "http://a/g" + "/../g" = "http://a/g" + "g." = "http://a/b/c/g." + ".g" = "http://a/b/c/.g" + "g.." = "http://a/b/c/g.." + "..g" = "http://a/b/c/..g" + + Less likely are cases where the relative reference uses unnecessary + or nonsensical forms of the "." and ".." complete path segments. + + "./../g" = "http://a/b/g" + "./g/." = "http://a/b/c/g/" + "g/./h" = "http://a/b/c/g/h" + "g/../h" = "http://a/b/c/h" + "g;x=1/./y" = "http://a/b/c/g;x=1/y" + "g;x=1/../y" = "http://a/b/c/y" + + Some applications fail to separate the reference's query and/or + fragment components from the path component before merging it with + the base path and removing dot-segments. This error is rarely + noticed, as typical usage of a fragment never includes the hierarchy + ("/") character and the query component is not normally used within + relative references. + + "g?y/./x" = "http://a/b/c/g?y/./x" + "g?y/../x" = "http://a/b/c/g?y/../x" + "g#s/./x" = "http://a/b/c/g#s/./x" + "g#s/../x" = "http://a/b/c/g#s/../x" + + Some parsers allow the scheme name to be present in a relative + reference if it is the same as the base URI scheme. This is + considered to be a loophole in prior specifications of partial URI + [RFC1630]. Its use should be avoided but is allowed for backward + compatibility. + + "http:g" = "http:g" ; for strict parsers + / "http://a/b/c/g" ; for backward compatibility + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 37] + +RFC 3986 URI Generic Syntax January 2005 + + +6. Normalization and Comparison + + One of the most common operations on URIs is simple comparison: + determining whether two URIs are equivalent without using the URIs to + access their respective resource(s). A comparison is performed every + time a response cache is accessed, a browser checks its history to + color a link, or an XML parser processes tags within a namespace. + Extensive normalization prior to comparison of URIs is often used by + spiders and indexing engines to prune a search space or to reduce + duplication of request actions and response storage. + + URI comparison is performed for some particular purpose. Protocols + or implementations that compare URIs for different purposes will + often be subject to differing design trade-offs in regards to how + much effort should be spent in reducing aliased identifiers. This + section describes various methods that may be used to compare URIs, + the trade-offs between them, and the types of applications that might + use them. + +6.1. Equivalence + + Because URIs exist to identify resources, presumably they should be + considered equivalent when they identify the same resource. However, + this definition of equivalence is not of much practical use, as there + is no way for an implementation to compare two resources unless it + has full knowledge or control of them. For this reason, + determination of equivalence or difference of URIs is based on string + comparison, perhaps augmented by reference to additional rules + provided by URI scheme definitions. We use the terms "different" and + "equivalent" to describe the possible outcomes of such comparisons, + but there are many application-dependent versions of equivalence. + + Even though it is possible to determine that two URIs are equivalent, + URI comparison is not sufficient to determine whether two URIs + identify different resources. For example, an owner of two different + domain names could decide to serve the same resource from both, + resulting in two different URIs. Therefore, comparison methods are + designed to minimize false negatives while strictly avoiding false + positives. + + In testing for equivalence, applications should not directly compare + relative references; the references should be converted to their + respective target URIs before comparison. When URIs are compared to + select (or avoid) a network action, such as retrieval of a + representation, fragment components (if any) should be excluded from + the comparison. + + + + + +Berners-Lee, et al. Standards Track [Page 38] + +RFC 3986 URI Generic Syntax January 2005 + + +6.2. Comparison Ladder + + A variety of methods are used in practice to test URI equivalence. + These methods fall into a range, distinguished by the amount of + processing required and the degree to which the probability of false + negatives is reduced. As noted above, false negatives cannot be + eliminated. In practice, their probability can be reduced, but this + reduction requires more processing and is not cost-effective for all + applications. + + If this range of comparison practices is considered as a ladder, the + following discussion will climb the ladder, starting with practices + that are cheap but have a relatively higher chance of producing false + negatives, and proceeding to those that have higher computational + cost and lower risk of false negatives. + +6.2.1. Simple String Comparison + + If two URIs, when considered as character strings, are identical, + then it is safe to conclude that they are equivalent. This type of + equivalence test has very low computational cost and is in wide use + in a variety of applications, particularly in the domain of parsing. + + Testing strings for equivalence requires some basic precautions. + This procedure is often referred to as "bit-for-bit" or + "byte-for-byte" comparison, which is potentially misleading. Testing + strings for equality is normally based on pair comparison of the + characters that make up the strings, starting from the first and + proceeding until both strings are exhausted and all characters are + found to be equal, until a pair of characters compares unequal, or + until one of the strings is exhausted before the other. + + This character comparison requires that each pair of characters be + put in comparable form. For example, should one URI be stored in a + byte array in EBCDIC encoding and the second in a Java String object + (UTF-16), bit-for-bit comparisons applied naively will produce + errors. It is better to speak of equality on a character-for- + character basis rather than on a byte-for-byte or bit-for-bit basis. + In practical terms, character-by-character comparisons should be done + codepoint-by-codepoint after conversion to a common character + encoding. + + False negatives are caused by the production and use of URI aliases. + Unnecessary aliases can be reduced, regardless of the comparison + method, by consistently providing URI references in an already- + normalized form (i.e., a form identical to what would be produced + after normalization is applied, as described below). + + + + +Berners-Lee, et al. Standards Track [Page 39] + +RFC 3986 URI Generic Syntax January 2005 + + + Protocols and data formats often limit some URI comparisons to simple + string comparison, based on the theory that people and + implementations will, in their own best interest, be consistent in + providing URI references, or at least consistent enough to negate any + efficiency that might be obtained from further normalization. + +6.2.2. Syntax-Based Normalization + + Implementations may use logic based on the definitions provided by + this specification to reduce the probability of false negatives. + This processing is moderately higher in cost than character-for- + character string comparison. For example, an application using this + approach could reasonably consider the following two URIs equivalent: + + example://a/b/c/%7Bfoo%7D + eXAMPLE://a/./b/../b/%63/%7bfoo%7d + + Web user agents, such as browsers, typically apply this type of URI + normalization when determining whether a cached response is + available. Syntax-based normalization includes such techniques as + case normalization, percent-encoding normalization, and removal of + dot-segments. + +6.2.2.1. Case Normalization + + For all URIs, the hexadecimal digits within a percent-encoding + triplet (e.g., "%3a" versus "%3A") are case-insensitive and therefore + should be normalized to use uppercase letters for the digits A-F. + + When a URI uses components of the generic syntax, the component + syntax equivalence rules always apply; namely, that the scheme and + host are case-insensitive and therefore should be normalized to + lowercase. For example, the URI <HTTP://www.EXAMPLE.com/> is + equivalent to <http://www.example.com/>. The other generic syntax + components are assumed to be case-sensitive unless specifically + defined otherwise by the scheme (see Section 6.2.3). + +6.2.2.2. Percent-Encoding Normalization + + The percent-encoding mechanism (Section 2.1) is a frequent source of + variance among otherwise identical URIs. In addition to the case + normalization issue noted above, some URI producers percent-encode + octets that do not require percent-encoding, resulting in URIs that + are equivalent to their non-encoded counterparts. These URIs should + be normalized by decoding any percent-encoded octet that corresponds + to an unreserved character, as described in Section 2.3. + + + + + +Berners-Lee, et al. Standards Track [Page 40] + +RFC 3986 URI Generic Syntax January 2005 + + +6.2.2.3. Path Segment Normalization + + The complete path segments "." and ".." are intended only for use + within relative references (Section 4.1) and are removed as part of + the reference resolution process (Section 5.2). However, some + deployed implementations incorrectly assume that reference resolution + is not necessary when the reference is already a URI and thus fail to + remove dot-segments when they occur in non-relative paths. URI + normalizers should remove dot-segments by applying the + remove_dot_segments algorithm to the path, as described in + Section 5.2.4. + +6.2.3. Scheme-Based Normalization + + The syntax and semantics of URIs vary from scheme to scheme, as + described by the defining specification for each scheme. + Implementations may use scheme-specific rules, at further processing + cost, to reduce the probability of false negatives. For example, + because the "http" scheme makes use of an authority component, has a + default port of "80", and defines an empty path to be equivalent to + "/", the following four URIs are equivalent: + + http://example.com + http://example.com/ + http://example.com:/ + http://example.com:80/ + + In general, a URI that uses the generic syntax for authority with an + empty path should be normalized to a path of "/". Likewise, an + explicit ":port", for which the port is empty or the default for the + scheme, is equivalent to one where the port and its ":" delimiter are + elided and thus should be removed by scheme-based normalization. For + example, the second URI above is the normal form for the "http" + scheme. + + Another case where normalization varies by scheme is in the handling + of an empty authority component or empty host subcomponent. For many + scheme specifications, an empty authority or host is considered an + error; for others, it is considered equivalent to "localhost" or the + end-user's host. When a scheme defines a default for authority and a + URI reference to that default is desired, the reference should be + normalized to an empty authority for the sake of uniformity, brevity, + and internationalization. If, however, either the userinfo or port + subcomponents are non-empty, then the host should be given explicitly + even if it matches the default. + + Normalization should not remove delimiters when their associated + component is empty unless licensed to do so by the scheme + + + +Berners-Lee, et al. Standards Track [Page 41] + +RFC 3986 URI Generic Syntax January 2005 + + + specification. For example, the URI "http://example.com/?" cannot be + assumed to be equivalent to any of the examples above. Likewise, the + presence or absence of delimiters within a userinfo subcomponent is + usually significant to its interpretation. The fragment component is + not subject to any scheme-based normalization; thus, two URIs that + differ only by the suffix "#" are considered different regardless of + the scheme. + + Some schemes define additional subcomponents that consist of case- + insensitive data, giving an implicit license to normalizers to + convert this data to a common case (e.g., all lowercase). For + example, URI schemes that define a subcomponent of path to contain an + Internet hostname, such as the "mailto" URI scheme, cause that + subcomponent to be case-insensitive and thus subject to case + normalization (e.g., "mailto:[email protected]" is equivalent to + "mailto:[email protected]", even though the generic syntax considers + the path component to be case-sensitive). + + Other scheme-specific normalizations are possible. + +6.2.4. Protocol-Based Normalization + + Substantial effort to reduce the incidence of false negatives is + often cost-effective for web spiders. Therefore, they implement even + more aggressive techniques in URI comparison. For example, if they + observe that a URI such as + + http://example.com/data + + redirects to a URI differing only in the trailing slash + + http://example.com/data/ + + they will likely regard the two as equivalent in the future. This + kind of technique is only appropriate when equivalence is clearly + indicated by both the result of accessing the resources and the + common conventions of their scheme's dereference algorithm (in this + case, use of redirection by HTTP origin servers to avoid problems + with relative references). + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 42] + +RFC 3986 URI Generic Syntax January 2005 + + +7. Security Considerations + + A URI does not in itself pose a security threat. However, as URIs + are often used to provide a compact set of instructions for access to + network resources, care must be taken to properly interpret the data + within a URI, to prevent that data from causing unintended access, + and to avoid including data that should not be revealed in plain + text. + +7.1. Reliability and Consistency + + There is no guarantee that once a URI has been used to retrieve + information, the same information will be retrievable by that URI in + the future. Nor is there any guarantee that the information + retrievable via that URI in the future will be observably similar to + that retrieved in the past. The URI syntax does not constrain how a + given scheme or authority apportions its namespace or maintains it + over time. Such guarantees can only be obtained from the person(s) + controlling that namespace and the resource in question. A specific + URI scheme may define additional semantics, such as name persistence, + if those semantics are required of all naming authorities for that + scheme. + +7.2. Malicious Construction + + It is sometimes possible to construct a URI so that an attempt to + perform a seemingly harmless, idempotent operation, such as the + retrieval of a representation, will in fact cause a possibly damaging + remote operation. The unsafe URI is typically constructed by + specifying a port number other than that reserved for the network + protocol in question. The client unwittingly contacts a site running + a different protocol service, and data within the URI contains + instructions that, when interpreted according to this other protocol, + cause an unexpected operation. A frequent example of such abuse has + been the use of a protocol-based scheme with a port component of + "25", thereby fooling user agent software into sending an unintended + or impersonating message via an SMTP server. + + Applications should prevent dereference of a URI that specifies a TCP + port number within the "well-known port" range (0 - 1023) unless the + protocol being used to dereference that URI is compatible with the + protocol expected on that well-known port. Although IANA maintains a + registry of well-known ports, applications should make such + restrictions user-configurable to avoid preventing the deployment of + new services. + + + + + + +Berners-Lee, et al. Standards Track [Page 43] + +RFC 3986 URI Generic Syntax January 2005 + + + When a URI contains percent-encoded octets that match the delimiters + for a given resolution or dereference protocol (for example, CR and + LF characters for the TELNET protocol), these percent-encodings must + not be decoded before transmission across that protocol. Transfer of + the percent-encoding, which might violate the protocol, is less + harmful than allowing decoded octets to be interpreted as additional + operations or parameters, perhaps triggering an unexpected and + possibly harmful remote operation. + +7.3. Back-End Transcoding + + When a URI is dereferenced, the data within it is often parsed by + both the user agent and one or more servers. In HTTP, for example, a + typical user agent will parse a URI into its five major components, + access the authority's server, and send it the data within the + authority, path, and query components. A typical server will take + that information, parse the path into segments and the query into + key/value pairs, and then invoke implementation-specific handlers to + respond to the request. As a result, a common security concern for + server implementations that handle a URI, either as a whole or split + into separate components, is proper interpretation of the octet data + represented by the characters and percent-encodings within that URI. + + Percent-encoded octets must be decoded at some point during the + dereference process. Applications must split the URI into its + components and subcomponents prior to decoding the octets, as + otherwise the decoded octets might be mistaken for delimiters. + Security checks of the data within a URI should be applied after + decoding the octets. Note, however, that the "%00" percent-encoding + (NUL) may require special handling and should be rejected if the + application is not expecting to receive raw data within a component. + + Special care should be taken when the URI path interpretation process + involves the use of a back-end file system or related system + functions. File systems typically assign an operational meaning to + special characters, such as the "/", "\", ":", "[", and "]" + characters, and to special device names like ".", "..", "...", "aux", + "lpt", etc. In some cases, merely testing for the existence of such + a name will cause the operating system to pause or invoke unrelated + system calls, leading to significant security concerns regarding + denial of service and unintended data transfer. It would be + impossible for this specification to list all such significant + characters and device names. Implementers should research the + reserved names and characters for the types of storage device that + may be attached to their applications and restrict the use of data + obtained from URI components accordingly. + + + + + +Berners-Lee, et al. Standards Track [Page 44] + +RFC 3986 URI Generic Syntax January 2005 + + +7.4. Rare IP Address Formats + + Although the URI syntax for IPv4address only allows the common + dotted-decimal form of IPv4 address literal, many implementations + that process URIs make use of platform-dependent system routines, + such as gethostbyname() and inet_aton(), to translate the string + literal to an actual IP address. Unfortunately, such system routines + often allow and process a much larger set of formats than those + described in Section 3.2.2. + + For example, many implementations allow dotted forms of three + numbers, wherein the last part is interpreted as a 16-bit quantity + and placed in the right-most two bytes of the network address (e.g., + a Class B network). Likewise, a dotted form of two numbers means + that the last part is interpreted as a 24-bit quantity and placed in + the right-most three bytes of the network address (Class A), and a + single number (without dots) is interpreted as a 32-bit quantity and + stored directly in the network address. Adding further to the + confusion, some implementations allow each dotted part to be + interpreted as decimal, octal, or hexadecimal, as specified in the C + language (i.e., a leading 0x or 0X implies hexadecimal; a leading 0 + implies octal; otherwise, the number is interpreted as decimal). + + These additional IP address formats are not allowed in the URI syntax + due to differences between platform implementations. However, they + can become a security concern if an application attempts to filter + access to resources based on the IP address in string literal format. + If this filtering is performed, literals should be converted to + numeric form and filtered based on the numeric value, and not on a + prefix or suffix of the string form. + +7.5. Sensitive Information + + URI producers should not provide a URI that contains a username or + password that is intended to be secret. URIs are frequently + displayed by browsers, stored in clear text bookmarks, and logged by + user agent history and intermediary applications (proxies). A + password appearing within the userinfo component is deprecated and + should be considered an error (or simply ignored) except in those + rare cases where the 'password' parameter is intended to be public. + +7.6. Semantic Attacks + + Because the userinfo subcomponent is rarely used and appears before + the host in the authority component, it can be used to construct a + URI intended to mislead a human user by appearing to identify one + (trusted) naming authority while actually identifying a different + authority hidden behind the noise. For example + + + +Berners-Lee, et al. Standards Track [Page 45] + +RFC 3986 URI Generic Syntax January 2005 + + + ftp://cnn.example.com&[email protected]/top_story.htm + + might lead a human user to assume that the host is 'cnn.example.com', + whereas it is actually '10.0.0.1'. Note that a misleading userinfo + subcomponent could be much longer than the example above. + + A misleading URI, such as that above, is an attack on the user's + preconceived notions about the meaning of a URI rather than an attack + on the software itself. User agents may be able to reduce the impact + of such attacks by distinguishing the various components of the URI + when they are rendered, such as by using a different color or tone to + render userinfo if any is present, though there is no panacea. More + information on URI-based semantic attacks can be found in [Siedzik]. + +8. IANA Considerations + + URI scheme names, as defined by <scheme> in Section 3.1, form a + registered namespace that is managed by IANA according to the + procedures defined in [BCP35]. No IANA actions are required by this + document. + +9. Acknowledgements + + This specification is derived from RFC 2396 [RFC2396], RFC 1808 + [RFC1808], and RFC 1738 [RFC1738]; the acknowledgements in those + documents still apply. It also incorporates the update (with + corrections) for IPv6 literals in the host syntax, as defined by + Robert M. Hinden, Brian E. Carpenter, and Larry Masinter in + [RFC2732]. In addition, contributions by Gisle Aas, Reese Anschultz, + Daniel Barclay, Tim Bray, Mike Brown, Rob Cameron, Jeremy Carroll, + Dan Connolly, Adam M. Costello, John Cowan, Jason Diamond, Martin + Duerst, Stefan Eissing, Clive D.W. Feather, Al Gilman, Tony Hammond, + Elliotte Harold, Pat Hayes, Henry Holtzman, Ian B. Jacobs, Michael + Kay, John C. Klensin, Graham Klyne, Dan Kohn, Bruce Lilly, Andrew + Main, Dave McAlpin, Ira McDonald, Michael Mealling, Ray Merkert, + Stephen Pollei, Julian Reschke, Tomas Rokicki, Miles Sabin, Kai + Schaetzl, Mark Thomson, Ronald Tschalaer, Norm Walsh, Marc Warne, + Stuart Williams, and Henry Zongaro are gratefully acknowledged. + +10. References + +10.1. Normative References + + [ASCII] American National Standards Institute, "Coded Character + Set -- 7-bit American Standard Code for Information + Interchange", ANSI X3.4, 1986. + + + + + +Berners-Lee, et al. Standards Track [Page 46] + +RFC 3986 URI Generic Syntax January 2005 + + + [RFC2234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [STD63] Yergeau, F., "UTF-8, a transformation format of + ISO 10646", STD 63, RFC 3629, November 2003. + + [UCS] International Organization for Standardization, + "Information Technology - Universal Multiple-Octet Coded + Character Set (UCS)", ISO/IEC 10646:2003, December 2003. + +10.2. Informative References + + [BCP19] Freed, N. and J. Postel, "IANA Charset Registration + Procedures", BCP 19, RFC 2978, October 2000. + + [BCP35] Petke, R. and I. King, "Registration Procedures for URL + Scheme Names", BCP 35, RFC 2717, November 1999. + + [RFC0952] Harrenstien, K., Stahl, M., and E. Feinler, "DoD Internet + host table specification", RFC 952, October 1985. + + [RFC1034] Mockapetris, P., "Domain names - concepts and facilities", + STD 13, RFC 1034, November 1987. + + [RFC1123] Braden, R., "Requirements for Internet Hosts - Application + and Support", STD 3, RFC 1123, October 1989. + + [RFC1535] Gavron, E., "A Security Problem and Proposed Correction + With Widely Deployed DNS Software", RFC 1535, + October 1993. + + [RFC1630] Berners-Lee, T., "Universal Resource Identifiers in WWW: A + Unifying Syntax for the Expression of Names and Addresses + of Objects on the Network as used in the World-Wide Web", + RFC 1630, June 1994. + + [RFC1736] Kunze, J., "Functional Recommendations for Internet + Resource Locators", RFC 1736, February 1995. + + [RFC1737] Sollins, K. and L. Masinter, "Functional Requirements for + Uniform Resource Names", RFC 1737, December 1994. + + [RFC1738] Berners-Lee, T., Masinter, L., and M. McCahill, "Uniform + Resource Locators (URL)", RFC 1738, December 1994. + + [RFC1808] Fielding, R., "Relative Uniform Resource Locators", + RFC 1808, June 1995. + + + + +Berners-Lee, et al. Standards Track [Page 47] + +RFC 3986 URI Generic Syntax January 2005 + + + [RFC2046] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part Two: Media Types", RFC 2046, + November 1996. + + [RFC2141] Moats, R., "URN Syntax", RFC 2141, May 1997. + + [RFC2396] Berners-Lee, T., Fielding, R., and L. Masinter, "Uniform + Resource Identifiers (URI): Generic Syntax", RFC 2396, + August 1998. + + [RFC2518] Goland, Y., Whitehead, E., Faizi, A., Carter, S., and D. + Jensen, "HTTP Extensions for Distributed Authoring -- + WEBDAV", RFC 2518, February 1999. + + [RFC2557] Palme, J., Hopmann, A., and N. Shelness, "MIME + Encapsulation of Aggregate Documents, such as HTML + (MHTML)", RFC 2557, March 1999. + + [RFC2718] Masinter, L., Alvestrand, H., Zigmond, D., and R. Petke, + "Guidelines for new URL Schemes", RFC 2718, November 1999. + + [RFC2732] Hinden, R., Carpenter, B., and L. Masinter, "Format for + Literal IPv6 Addresses in URL's", RFC 2732, December 1999. + + [RFC3305] Mealling, M. and R. Denenberg, "Report from the Joint + W3C/IETF URI Planning Interest Group: Uniform Resource + Identifiers (URIs), URLs, and Uniform Resource Names + (URNs): Clarifications and Recommendations", RFC 3305, + August 2002. + + [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, + "Internationalizing Domain Names in Applications (IDNA)", + RFC 3490, March 2003. + + [RFC3513] Hinden, R. and S. Deering, "Internet Protocol Version 6 + (IPv6) Addressing Architecture", RFC 3513, April 2003. + + [Siedzik] Siedzik, R., "Semantic Attacks: What's in a URL?", + April 2001, <http://www.giac.org/practical/gsec/ + Richard_Siedzik_GSEC.pdf>. + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 48] + +RFC 3986 URI Generic Syntax January 2005 + + +Appendix A. Collected ABNF for URI + + URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] + + hier-part = "//" authority path-abempty + / path-absolute + / path-rootless + / path-empty + + URI-reference = URI / relative-ref + + absolute-URI = scheme ":" hier-part [ "?" query ] + + relative-ref = relative-part [ "?" query ] [ "#" fragment ] + + relative-part = "//" authority path-abempty + / path-absolute + / path-noscheme + / path-empty + + scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + + authority = [ userinfo "@" ] host [ ":" port ] + userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + host = IP-literal / IPv4address / reg-name + port = *DIGIT + + IP-literal = "[" ( IPv6address / IPvFuture ) "]" + + IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) + + IPv6address = 6( h16 ":" ) ls32 + / "::" 5( h16 ":" ) ls32 + / [ h16 ] "::" 4( h16 ":" ) ls32 + / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + / [ *4( h16 ":" ) h16 ] "::" ls32 + / [ *5( h16 ":" ) h16 ] "::" h16 + / [ *6( h16 ":" ) h16 ] "::" + + h16 = 1*4HEXDIG + ls32 = ( h16 ":" h16 ) / IPv4address + IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + + + + + + + +Berners-Lee, et al. Standards Track [Page 49] + +RFC 3986 URI Generic Syntax January 2005 + + + dec-octet = DIGIT ; 0-9 + / %x31-39 DIGIT ; 10-99 + / "1" 2DIGIT ; 100-199 + / "2" %x30-34 DIGIT ; 200-249 + / "25" %x30-35 ; 250-255 + + reg-name = *( unreserved / pct-encoded / sub-delims ) + + path = path-abempty ; begins with "/" or is empty + / path-absolute ; begins with "/" but not "//" + / path-noscheme ; begins with a non-colon segment + / path-rootless ; begins with a segment + / path-empty ; zero characters + + path-abempty = *( "/" segment ) + path-absolute = "/" [ segment-nz *( "/" segment ) ] + path-noscheme = segment-nz-nc *( "/" segment ) + path-rootless = segment-nz *( "/" segment ) + path-empty = 0<pchar> + + segment = *pchar + segment-nz = 1*pchar + segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + ; non-zero-length segment without any colon ":" + + pchar = unreserved / pct-encoded / sub-delims / ":" / "@" + + query = *( pchar / "/" / "?" ) + + fragment = *( pchar / "/" / "?" ) + + pct-encoded = "%" HEXDIG HEXDIG + + unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + reserved = gen-delims / sub-delims + gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + / "*" / "+" / "," / ";" / "=" + +Appendix B. Parsing a URI Reference with a Regular Expression + + As the "first-match-wins" algorithm is identical to the "greedy" + disambiguation method used by POSIX regular expressions, it is + natural and commonplace to use a regular expression for parsing the + potential five components of a URI reference. + + The following line is the regular expression for breaking-down a + well-formed URI reference into its components. + + + +Berners-Lee, et al. Standards Track [Page 50] + +RFC 3986 URI Generic Syntax January 2005 + + + ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? + 12 3 4 5 6 7 8 9 + + The numbers in the second line above are only to assist readability; + they indicate the reference points for each subexpression (i.e., each + paired parenthesis). We refer to the value matched for subexpression + <n> as $<n>. For example, matching the above expression to + + http://www.ics.uci.edu/pub/ietf/uri/#Related + + results in the following subexpression matches: + + $1 = http: + $2 = http + $3 = //www.ics.uci.edu + $4 = www.ics.uci.edu + $5 = /pub/ietf/uri/ + $6 = <undefined> + $7 = <undefined> + $8 = #Related + $9 = Related + + where <undefined> indicates that the component is not present, as is + the case for the query component in the above example. Therefore, we + can determine the value of the five components as + + scheme = $2 + authority = $4 + path = $5 + query = $7 + fragment = $9 + + Going in the opposite direction, we can recreate a URI reference from + its components by using the algorithm of Section 5.3. + +Appendix C. Delimiting a URI in Context + + URIs are often transmitted through formats that do not provide a + clear context for their interpretation. For example, there are many + occasions when a URI is included in plain text; examples include text + sent in email, USENET news, and on printed paper. In such cases, it + is important to be able to delimit the URI from the rest of the text, + and in particular from punctuation marks that might be mistaken for + part of the URI. + + In practice, URIs are delimited in a variety of ways, but usually + within double-quotes "http://example.com/", angle brackets + <http://example.com/>, or just by using whitespace: + + + +Berners-Lee, et al. Standards Track [Page 51] + +RFC 3986 URI Generic Syntax January 2005 + + + http://example.com/ + + These wrappers do not form part of the URI. + + In some cases, extra whitespace (spaces, line-breaks, tabs, etc.) may + have to be added to break a long URI across lines. The whitespace + should be ignored when the URI is extracted. + + No whitespace should be introduced after a hyphen ("-") character. + Because some typesetters and printers may (erroneously) introduce a + hyphen at the end of line when breaking it, the interpreter of a URI + containing a line break immediately after a hyphen should ignore all + whitespace around the line break and should be aware that the hyphen + may or may not actually be part of the URI. + + Using <> angle brackets around each URI is especially recommended as + a delimiting style for a reference that contains embedded whitespace. + + The prefix "URL:" (with or without a trailing space) was formerly + recommended as a way to help distinguish a URI from other bracketed + designators, though it is not commonly used in practice and is no + longer recommended. + + For robustness, software that accepts user-typed URI should attempt + to recognize and strip both delimiters and embedded whitespace. + + For example, the text + + Yes, Jim, I found it under "http://www.w3.org/Addressing/", + but you can probably pick it up from <ftp://foo.example. + com/rfc/>. Note the warning in <http://www.ics.uci.edu/pub/ + ietf/uri/historical.html#WARNING>. + + contains the URI references + + http://www.w3.org/Addressing/ + ftp://foo.example.com/rfc/ + http://www.ics.uci.edu/pub/ietf/uri/historical.html#WARNING + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 52] + +RFC 3986 URI Generic Syntax January 2005 + + +Appendix D. Changes from RFC 2396 + +D.1. Additions + + An ABNF rule for URI has been introduced to correspond to one common + usage of the term: an absolute URI with optional fragment. + + IPv6 (and later) literals have been added to the list of possible + identifiers for the host portion of an authority component, as + described by [RFC2732], with the addition of "[" and "]" to the + reserved set and a version flag to anticipate future versions of IP + literals. Square brackets are now specified as reserved within the + authority component and are not allowed outside their use as + delimiters for an IP literal within host. In order to make this + change without changing the technical definition of the path, query, + and fragment components, those rules were redefined to directly + specify the characters allowed. + + As [RFC2732] defers to [RFC3513] for definition of an IPv6 literal + address, which, unfortunately, lacks an ABNF description of + IPv6address, we created a new ABNF rule for IPv6address that matches + the text representations defined by Section 2.2 of [RFC3513]. + Likewise, the definition of IPv4address has been improved in order to + limit each decimal octet to the range 0-255. + + Section 6, on URI normalization and comparison, has been completely + rewritten and extended by using input from Tim Bray and discussion + within the W3C Technical Architecture Group. + +D.2. Modifications + + The ad-hoc BNF syntax of RFC 2396 has been replaced with the ABNF of + [RFC2234]. This change required all rule names that formerly + included underscore characters to be renamed with a dash instead. In + addition, a number of syntax rules have been eliminated or simplified + to make the overall grammar more comprehensible. Specifications that + refer to the obsolete grammar rules may be understood by replacing + those rules according to the following table: + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 53] + +RFC 3986 URI Generic Syntax January 2005 + + + +----------------+--------------------------------------------------+ + | obsolete rule | translation | + +----------------+--------------------------------------------------+ + | absoluteURI | absolute-URI | + | relativeURI | relative-part [ "?" query ] | + | hier_part | ( "//" authority path-abempty / | + | | path-absolute ) [ "?" query ] | + | | | + | opaque_part | path-rootless [ "?" query ] | + | net_path | "//" authority path-abempty | + | abs_path | path-absolute | + | rel_path | path-rootless | + | rel_segment | segment-nz-nc | + | reg_name | reg-name | + | server | authority | + | hostport | host [ ":" port ] | + | hostname | reg-name | + | path_segments | path-abempty | + | param | *<pchar excluding ";"> | + | | | + | uric | unreserved / pct-encoded / ";" / "?" / ":" | + | | / "@" / "&" / "=" / "+" / "$" / "," / "/" | + | | | + | uric_no_slash | unreserved / pct-encoded / ";" / "?" / ":" | + | | / "@" / "&" / "=" / "+" / "$" / "," | + | | | + | mark | "-" / "_" / "." / "!" / "~" / "*" / "'" | + | | / "(" / ")" | + | | | + | escaped | pct-encoded | + | hex | HEXDIG | + | alphanum | ALPHA / DIGIT | + +----------------+--------------------------------------------------+ + + Use of the above obsolete rules for the definition of scheme-specific + syntax is deprecated. + + Section 2, on characters, has been rewritten to explain what + characters are reserved, when they are reserved, and why they are + reserved, even when they are not used as delimiters by the generic + syntax. The mark characters that are typically unsafe to decode, + including the exclamation mark ("!"), asterisk ("*"), single-quote + ("'"), and open and close parentheses ("(" and ")"), have been moved + to the reserved set in order to clarify the distinction between + reserved and unreserved and, hopefully, to answer the most common + question of scheme designers. Likewise, the section on + percent-encoded characters has been rewritten, and URI normalizers + are now given license to decode any percent-encoded octets + + + +Berners-Lee, et al. Standards Track [Page 54] + +RFC 3986 URI Generic Syntax January 2005 + + + corresponding to unreserved characters. In general, the terms + "escaped" and "unescaped" have been replaced with "percent-encoded" + and "decoded", respectively, to reduce confusion with other forms of + escape mechanisms. + + The ABNF for URI and URI-reference has been redesigned to make them + more friendly to LALR parsers and to reduce complexity. As a result, + the layout form of syntax description has been removed, along with + the uric, uric_no_slash, opaque_part, net_path, abs_path, rel_path, + path_segments, rel_segment, and mark rules. All references to + "opaque" URIs have been replaced with a better description of how the + path component may be opaque to hierarchy. The relativeURI rule has + been replaced with relative-ref to avoid unnecessary confusion over + whether they are a subset of URI. The ambiguity regarding the + parsing of URI-reference as a URI or a relative-ref with a colon in + the first segment has been eliminated through the use of five + separate path matching rules. + + The fragment identifier has been moved back into the section on + generic syntax components and within the URI and relative-ref rules, + though it remains excluded from absolute-URI. The number sign ("#") + character has been moved back to the reserved set as a result of + reintegrating the fragment syntax. + + The ABNF has been corrected to allow the path component to be empty. + This also allows an absolute-URI to consist of nothing after the + "scheme:", as is present in practice with the "dav:" namespace + [RFC2518] and with the "about:" scheme used internally by many WWW + browser implementations. The ambiguity regarding the boundary + between authority and path has been eliminated through the use of + five separate path matching rules. + + Registry-based naming authorities that use the generic syntax are now + defined within the host rule. This change allows current + implementations, where whatever name provided is simply fed to the + local name resolution mechanism, to be consistent with the + specification. It also removes the need to re-specify DNS name + formats here. Furthermore, it allows the host component to contain + percent-encoded octets, which is necessary to enable + internationalized domain names to be provided in URIs, processed in + their native character encodings at the application layers above URI + processing, and passed to an IDNA library as a registered name in the + UTF-8 character encoding. The server, hostport, hostname, + domainlabel, toplabel, and alphanum rules have been removed. + + The resolving relative references algorithm of [RFC2396] has been + rewritten with pseudocode for this revision to improve clarity and + fix the following issues: + + + +Berners-Lee, et al. Standards Track [Page 55] + +RFC 3986 URI Generic Syntax January 2005 + + + o [RFC2396] section 5.2, step 6a, failed to account for a base URI + with no path. + + o Restored the behavior of [RFC1808] where, if the reference + contains an empty path and a defined query component, the target + URI inherits the base URI's path component. + + o The determination of whether a URI reference is a same-document + reference has been decoupled from the URI parser, simplifying the + URI processing interface within applications in a way consistent + with the internal architecture of deployed URI processing + implementations. The determination is now based on comparison to + the base URI after transforming a reference to absolute form, + rather than on the format of the reference itself. This change + may result in more references being considered "same-document" + under this specification than there would be under the rules given + in RFC 2396, especially when normalization is used to reduce + aliases. However, it does not change the status of existing + same-document references. + + o Separated the path merge routine into two routines: merge, for + describing combination of the base URI path with a relative-path + reference, and remove_dot_segments, for describing how to remove + the special "." and ".." segments from a composed path. The + remove_dot_segments algorithm is now applied to all URI reference + paths in order to match common implementations and to improve the + normalization of URIs in practice. This change only impacts the + parsing of abnormal references and same-scheme references wherein + the base URI has a non-hierarchical path. + +Index + + A + ABNF 11 + absolute 27 + absolute-path 26 + absolute-URI 27 + access 9 + authority 17, 18 + + B + base URI 28 + + C + character encoding 4 + character 4 + characters 8, 11 + coded character set 4 + + + +Berners-Lee, et al. Standards Track [Page 56] + +RFC 3986 URI Generic Syntax January 2005 + + + D + dec-octet 20 + dereference 9 + dot-segments 23 + + F + fragment 16, 24 + + G + gen-delims 13 + generic syntax 6 + + H + h16 20 + hier-part 16 + hierarchical 10 + host 18 + + I + identifier 5 + IP-literal 19 + IPv4 20 + IPv4address 19, 20 + IPv6 19 + IPv6address 19, 20 + IPvFuture 19 + + L + locator 7 + ls32 20 + + M + merge 32 + + N + name 7 + network-path 26 + + P + path 16, 22, 26 + path-abempty 22 + path-absolute 22 + path-empty 22 + path-noscheme 22 + path-rootless 22 + path-abempty 16, 22, 26 + path-absolute 16, 22, 26 + path-empty 16, 22, 26 + + + +Berners-Lee, et al. Standards Track [Page 57] + +RFC 3986 URI Generic Syntax January 2005 + + + path-rootless 16, 22 + pchar 23 + pct-encoded 12 + percent-encoding 12 + port 22 + + Q + query 16, 23 + + R + reg-name 21 + registered name 20 + relative 10, 28 + relative-path 26 + relative-ref 26 + remove_dot_segments 33 + representation 9 + reserved 12 + resolution 9, 28 + resource 5 + retrieval 9 + + S + same-document 27 + sameness 9 + scheme 16, 17 + segment 22, 23 + segment-nz 23 + segment-nz-nc 23 + sub-delims 13 + suffix 27 + + T + transcription 8 + + U + uniform 4 + unreserved 13 + URI grammar + absolute-URI 27 + ALPHA 11 + authority 18 + CR 11 + dec-octet 20 + DIGIT 11 + DQUOTE 11 + fragment 24 + gen-delims 13 + + + +Berners-Lee, et al. Standards Track [Page 58] + +RFC 3986 URI Generic Syntax January 2005 + + + h16 20 + HEXDIG 11 + hier-part 16 + host 19 + IP-literal 19 + IPv4address 20 + IPv6address 20 + IPvFuture 19 + LF 11 + ls32 20 + OCTET 11 + path 22 + path-abempty 22 + path-absolute 22 + path-empty 22 + path-noscheme 22 + path-rootless 22 + pchar 23 + pct-encoded 12 + port 22 + query 24 + reg-name 21 + relative-ref 26 + reserved 13 + scheme 17 + segment 23 + segment-nz 23 + segment-nz-nc 23 + SP 11 + sub-delims 13 + unreserved 13 + URI 16 + URI-reference 25 + userinfo 18 + URI 16 + URI-reference 25 + URL 7 + URN 7 + userinfo 18 + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 59] + +RFC 3986 URI Generic Syntax January 2005 + + +Authors' Addresses + + Tim Berners-Lee + World Wide Web Consortium + Massachusetts Institute of Technology + 77 Massachusetts Avenue + Cambridge, MA 02139 + USA + + Phone: +1-617-253-5702 + Fax: +1-617-258-5999 + EMail: [email protected] + URI: http://www.w3.org/People/Berners-Lee/ + + + Roy T. Fielding + Day Software + 5251 California Ave., Suite 110 + Irvine, CA 92617 + USA + + Phone: +1-949-679-2960 + Fax: +1-949-679-2972 + EMail: [email protected] + URI: http://roy.gbiv.com/ + + + Larry Masinter + Adobe Systems Incorporated + 345 Park Ave + San Jose, CA 95110 + USA + + Phone: +1-408-536-3024 + EMail: [email protected] + URI: http://larry.masinter.net/ + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 60] + +RFC 3986 URI Generic Syntax January 2005 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2005). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the IETF's procedures with respect to rights in IETF Documents can + be found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at ietf- + + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + +Berners-Lee, et al. Standards Track [Page 61] + diff --git a/lib/inets/doc/src/ftp.xml b/lib/inets/doc/src/ftp.xml index ca902d8d9d..f8f11ec705 100644 --- a/lib/inets/doc/src/ftp.xml +++ b/lib/inets/doc/src/ftp.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="iso-8859-1" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> <header> <copyright> - <year>1997</year><year>2010</year> + <year>1997</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -141,11 +141,21 @@ <tag>{timeout, Timeout}</tag> <item> <marker id="timeout"></marker> - <p>Timeout = <c>integer() >= 0</c> </p> + <p>Timeout = <c>non_neg_integer()</c> </p> <p>Connection timeout. </p> <p>Default is 60000 (milliseconds). </p> </item> + <tag>{dtimeout, DTimeout}</tag> + <item> + <marker id="dtimeout"></marker> + <p>DTimeout = <c>non_neg_integer() | infinity</c> </p> + <p>Data Connect timeout. + The time the client will wait for the server to connect to the + data socket. </p> + <p>Default is infinity. </p> + </item> + <tag>{progress, Progress}</tag> <item> <marker id="progress"></marker> @@ -542,11 +552,12 @@ <v>verbose() = boolean() (defaults to false)</v> <v>debug() = disable | debug | trace (defaults to disable)</v> <!-- <v>open_options() = [open_option()]</v> --> - <v>open_option() = {ipfamily, ipfamily()} | {port, port()} | {mode, mode()} | {timeout, timeout()} | {progress, progress()}</v> + <v>open_option() = {ipfamily, ipfamily()} | {port, port()} | {mode, mode()} | {timeout, timeout()} | {dtimeout, dtimeout()} | {progress, progress()}</v> <v>ipfamily() = inet | inet6 | inet6fb4 (defaults to inet)</v> <v>port() = integer() > 0 (defaults to 21)</v> <v>mode() = active | passive (defaults to passive)</v> - <v>timeout() = integer() >= 0 (defaults to 60000 milliseconds)</v> + <v>timeout() = integer() > 0 (defaults to 60000 milliseconds)</v> + <v>dtimeout() = integer() > 0 | infinity (defaults to infinity)</v> <v>pogress() = ignore | {module(), function(), initial_data()} (defaults to ignore)</v> <v>module() = atom()</v> <v>function() = atom()</v> diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 87a8c173a5..59c2f8a2df 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -35,17 +35,22 @@ <section><title>Inets 5.8</title> <section><title>Improvements and New Features</title> +<!-- <p>-</p> +--> -<!-- <list> <item> - <p>[httpc|httpd] Added support for IPv6 with ssl. </p> - <p>Own Id: OTP-5566</p> + <p>[ftpc] Add a config option to specify a + <seealso marker="ftp#dtimeout">data connect timeout</seealso>. + That is how long the ftp client will wait for the server to connect + to the data socket. If this timeout occurs, an error will be + returned to the caller and the ftp client process will be + terminated. </p> + <p>Own Id: OTP-9545</p> </item> </list> ---> </section> @@ -94,6 +99,72 @@ </section> <!-- 5.8 --> + <section><title>Inets 5.7.2</title> + <section><title>Improvements and New Features</title> + <p>-</p> + +<!-- + <list> + <item> + <p>[httpc|httpd] Added support for IPv6 with ssl. </p> + <p>Own Id: OTP-5566</p> + </item> + + </list> +--> + + </section> + + <section> + <title>Incompatibilities</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[httpc] Deprecated interface module <c>http</c> has been removed. + It has (long) been replaced by http client interface module + <seealso marker="httpc#">httpc</seealso>. </p> + <p>Own Id: OTP-9359</p> + </item> + + <item> + <p>[httpc|httpd] The old ssl implementation (based on OpenSSL), + has been deprecated. The config option that specified usage of + this version of the ssl app, <c>ossl</c>, has been removed. </p> + <p>Own Id: OTP-9522</p> + </item> + + </list> + + </section> + + <section><title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[httpd] XSS prevention did not work for hex-encoded URL's. </p> + <p>Own Id: OTP-9655</p> + </item> + + <item> + <p>[httpd] GET request with malformed header date caused + server crash (non-fatal) with no reply to client. Will + now result in a reply with status code 400. </p> + <p>Own Id: OTP-9674</p> + <p>Aux Id: seq11936</p> + </item> + + </list> + </section> + + </section> <!-- 5.7.2 --> + + <section><title>Inets 5.7.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index ac72963347..b6da92947c 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -55,9 +55,10 @@ -include("ftp_internal.hrl"). %% Constante used in internal state definition --define(CONNECTION_TIMEOUT, 60*1000). --define(DEFAULT_MODE, passive). --define(PROGRESS_DEFAULT, ignore). +-define(CONNECTION_TIMEOUT, 60*1000). +-define(DATA_ACCEPT_TIMEOUT, infinity). +-define(DEFAULT_MODE, passive). +-define(PROGRESS_DEFAULT, ignore). %% Internal Constants -define(FTP_PORT, 21). @@ -88,7 +89,8 @@ %% data needed further on. caller = undefined, % term() ipfamily, % inet | inet6 | inet6fb4 - progress = ignore % ignore | pid() + progress = ignore, % ignore | pid() + dtimeout = ?DATA_ACCEPT_TIMEOUT % non_neg_integer() | infinity }). @@ -847,6 +849,7 @@ start_options(Options) -> %% host %% port %% timeout +%% dtimeout %% progress open_options(Options) -> ?fcrt("open_options", [{options, Options}]), @@ -875,7 +878,12 @@ open_options(Options) -> (_) -> false end, ValidateTimeout = - fun(Timeout) when is_integer(Timeout) andalso (Timeout > 0) -> true; + fun(Timeout) when is_integer(Timeout) andalso (Timeout >= 0) -> true; + (_) -> false + end, + ValidateDTimeout = + fun(DTimeout) when is_integer(DTimeout) andalso (DTimeout >= 0) -> true; + (infinity) -> true; (_) -> false end, ValidateProgress = @@ -893,6 +901,7 @@ open_options(Options) -> {port, ValidatePort, false, ?FTP_PORT}, {ipfamily, ValidateIpFamily, false, inet}, {timeout, ValidateTimeout, false, ?CONNECTION_TIMEOUT}, + {dtimeout, ValidateDTimeout, false, ?DATA_ACCEPT_TIMEOUT}, {progress, ValidateProgress, false, ?PROGRESS_DEFAULT}], validate_options(Options, ValidOptions, []). @@ -1037,13 +1046,15 @@ handle_call({_, {open, ip_comm, Opts}}, From, State) -> Mode = key_search(mode, Opts, ?DEFAULT_MODE), Port = key_search(port, Opts, ?FTP_PORT), Timeout = key_search(timeout, Opts, ?CONNECTION_TIMEOUT), + DTimeout = key_search(dtimeout, Opts, ?DATA_ACCEPT_TIMEOUT), Progress = key_search(progress, Opts, ignore), IpFamily = key_search(ipfamily, Opts, inet), - + State2 = State#state{client = From, mode = Mode, progress = progress(Progress), - ipfamily = IpFamily}, + ipfamily = IpFamily, + dtimeout = DTimeout}, ?fcrd("handle_call(open) -> setup ctrl connection with", [{host, Host}, {port, Port}, {timeout, Timeout}]), @@ -1064,11 +1075,13 @@ handle_call({_, {open, ip_comm, Host, Opts}}, From, State) -> Mode = key_search(mode, Opts, ?DEFAULT_MODE), Port = key_search(port, Opts, ?FTP_PORT), Timeout = key_search(timeout, Opts, ?CONNECTION_TIMEOUT), + DTimeout = key_search(dtimeout, Opts, ?DATA_ACCEPT_TIMEOUT), Progress = key_search(progress, Opts, ignore), State2 = State#state{client = From, mode = Mode, - progress = progress(Progress)}, + progress = progress(Progress), + dtimeout = DTimeout}, case setup_ctrl_connection(Host, Port, Timeout, State2) of {ok, State3, WaitTimeout} -> @@ -1657,9 +1670,19 @@ handle_ctrl_result({pos_compl, Lines}, %%-------------------------------------------------------------------------- %% Directory listing handle_ctrl_result({pos_prel, _}, #state{caller = {dir, Dir}} = State) -> - NewState = accept_data_connection(State), - activate_data_connection(NewState), - {noreply, NewState#state{caller = {handle_dir_result, Dir}}}; + case accept_data_connection(State) of + {ok, NewState} -> + activate_data_connection(NewState), + {noreply, NewState#state{caller = {handle_dir_result, Dir}}}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_result, Dir, Data}, client = From} @@ -1756,9 +1779,19 @@ handle_ctrl_result({Status, _}, %%-------------------------------------------------------------------------- %% File handling - recv_bin handle_ctrl_result({pos_prel, _}, #state{caller = recv_bin} = State) -> - NewState = accept_data_connection(State), - activate_data_connection(NewState), - {noreply, NewState}; + case accept_data_connection(State) of + {ok, NewState} -> + activate_data_connection(NewState), + {noreply, NewState}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({pos_compl, _}, #state{caller = {recv_bin, Data}, client = From} = State) -> @@ -1780,16 +1813,37 @@ handle_ctrl_result({Status, _}, #state{caller = {recv_bin, _}} = State) -> handle_ctrl_result({pos_prel, _}, #state{client = From, caller = start_chunk_transfer} = State) -> - NewState = accept_data_connection(State), - gen_server:reply(From, ok), - {noreply, NewState#state{chunk = true, client = undefined, - caller = undefined}}; + case accept_data_connection(State) of + {ok, NewState} -> + gen_server:reply(From, ok), + {noreply, NewState#state{chunk = true, client = undefined, + caller = undefined}}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; + %%-------------------------------------------------------------------------- %% File handling - recv_file handle_ctrl_result({pos_prel, _}, #state{caller = {recv_file, _}} = State) -> - NewState = accept_data_connection(State), - activate_data_connection(NewState), - {noreply, NewState}; + case accept_data_connection(State) of + {ok, NewState} -> + activate_data_connection(NewState), + {noreply, NewState}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({Status, _}, #state{caller = {recv_file, Fd}} = State) -> file_close(Fd), @@ -1800,17 +1854,38 @@ handle_ctrl_result({Status, _}, #state{caller = {recv_file, Fd}} = State) -> %% File handling - transfer_* handle_ctrl_result({pos_prel, _}, #state{caller = {transfer_file, Fd}} = State) -> - NewState = accept_data_connection(State), - send_file(Fd, NewState); + case accept_data_connection(State) of + {ok, NewState} -> + send_file(Fd, NewState); + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({pos_prel, _}, #state{caller = {transfer_data, Bin}} = State) -> - NewState = accept_data_connection(State), - send_data_message(NewState, Bin), - close_data_connection(NewState), - activate_ctrl_connection(NewState), - {noreply, NewState#state{caller = transfer_data_second_phase, - dsock = undefined}}; + case accept_data_connection(State) of + {ok, NewState} -> + send_data_message(NewState, Bin), + close_data_connection(NewState), + activate_ctrl_connection(NewState), + {noreply, NewState#state{caller = transfer_data_second_phase, + dsock = undefined}}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; + %%-------------------------------------------------------------------------- %% Default handle_ctrl_result({Status, Lines}, #state{client = From} = State) @@ -2009,16 +2084,20 @@ connect2(Host, Port, IpFam, Timeout) -> Error end. - -accept_data_connection(#state{mode = active, - dsock = {lsock, LSock}} = State) -> - {ok, Socket} = gen_tcp:accept(LSock), - gen_tcp:close(LSock), - State#state{dsock = Socket}; +accept_data_connection(#state{mode = active, + dtimeout = DTimeout, + dsock = {lsock, LSock}} = State) -> + case gen_tcp:accept(LSock, DTimeout) of + {ok, Socket} -> + gen_tcp:close(LSock), + {ok, State#state{dsock = Socket}}; + {error, Reason} -> + {error, {data_connect_failed, Reason}} + end; accept_data_connection(#state{mode = passive} = State) -> - State. + {ok, State}. send_ctrl_message(#state{csock = Socket, verbose = Verbose}, Message) -> %% io:format("send control message: ~n~p~n", [lists:flatten(Message)]), diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 75c26c63cc..d72c34fa6b 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -141,7 +141,9 @@ request(Url, Profile) -> request(Method, Request, HttpOptions, Options) -> request(Method, Request, HttpOptions, Options, default_profile()). -request(Method, {Url, Headers}, HTTPOptions, Options, Profile) +request(Method, + {Url, Headers}, + HTTPOptions, Options, Profile) when (Method =:= options) orelse (Method =:= get) orelse (Method =:= head) orelse @@ -154,15 +156,17 @@ request(Method, {Url, Headers}, HTTPOptions, Options, Profile) {http_options, HTTPOptions}, {options, Options}, {profile, Profile}]), - case http_uri:parse(Url) of + case http_uri:parse(Url, Options) of {error, Reason} -> {error, Reason}; - ParsedUrl -> + {ok, ParsedUrl} -> handle_request(Method, Url, ParsedUrl, Headers, [], [], HTTPOptions, Options, Profile) end; -request(Method, {Url,Headers,ContentType,Body}, HTTPOptions, Options, Profile) +request(Method, + {Url, Headers, ContentType, Body}, + HTTPOptions, Options, Profile) when ((Method =:= post) orelse (Method =:= put)) andalso (is_atom(Profile) orelse is_pid(Profile)) -> ?hcrt("request", [{method, Method}, @@ -173,10 +177,10 @@ request(Method, {Url,Headers,ContentType,Body}, HTTPOptions, Options, Profile) {http_options, HTTPOptions}, {options, Options}, {profile, Profile}]), - case http_uri:parse(Url) of + case http_uri:parse(Url, Options) of {error, Reason} -> {error, Reason}; - ParsedUrl -> + {ok, ParsedUrl} -> handle_request(Method, Url, ParsedUrl, Headers, ContentType, Body, HTTPOptions, Options, Profile) @@ -267,7 +271,10 @@ store_cookies(SetCookieHeaders, Url, Profile) {profile, Profile}]), try begin - {_, _, Host, Port, Path, _} = http_uri:parse(Url), + %% Since the Address part is not actually used + %% by the manager when storing cookies, we dont + %% care about ipv6-host-with-brackets. + {ok, {_, _, Host, Port, Path, _}} = http_uri:parse(Url), Address = {Host, Port}, ProfileName = profile_name(Profile), Cookies = httpc_cookie:cookies(SetCookieHeaders, Path, Host), @@ -464,6 +471,8 @@ handle_request(Method, Url, HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), + BracketedHost = proplists:get_value(ipv6_host_with_brackets, + Options), MaybeEscPath = maybe_encode_uri(HTTPOptions, Path), MaybeEscQuery = maybe_encode_uri(HTTPOptions, Query), AbsUri = maybe_encode_uri(HTTPOptions, Url), @@ -482,7 +491,8 @@ handle_request(Method, Url, stream = Stream, headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, - started = Started}, + started = Started, + ipv6_host_with_brackets = BracketedHost}, case httpc_manager:request(Request, profile_name(Profile)) of {ok, RequestId} -> @@ -739,14 +749,17 @@ request_options_defaults() -> error end, + VerifyBrackets = VerifyBoolean, + [ - {sync, true, VerifySync}, - {stream, none, VerifyStream}, - {body_format, string, VerifyBodyFormat}, - {full_result, true, VerifyFullResult}, - {headers_as_is, false, VerifyHeaderAsIs}, - {receiver, self(), VerifyReceiver}, - {socket_opts, undefined, VerifySocketOpts} + {sync, true, VerifySync}, + {stream, none, VerifyStream}, + {body_format, string, VerifyBodyFormat}, + {full_result, true, VerifyFullResult}, + {headers_as_is, false, VerifyHeaderAsIs}, + {receiver, self(), VerifyReceiver}, + {socket_opts, undefined, VerifySocketOpts}, + {ipv6_host_with_brackets, false, VerifyBrackets} ]. request_options(Options) -> diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl index 1d8a5b6a92..e4127d992d 100644 --- a/lib/inets/src/http_client/httpc_internal.hrl +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -90,25 +90,28 @@ %%% All data associated to a specific HTTP request -record(request, { - id, % ref() - Request Id - from, % pid() - Caller - redircount = 0,% Number of redirects made for this request - scheme, % http | https - address, % ({Host,Port}) Destination Host and Port - path, % string() - Path of parsed URL - pquery, % string() - Rest of parsed URL - method, % atom() - HTTP request Method - headers, % #http_request_h{} - content, % {ContentType, Body} - Current HTTP request - settings, % #http_options{} - User defined settings - abs_uri, % string() ex: "http://www.erlang.org" - userinfo, % string() - optinal "<userinfo>@<host>:<port>" - stream, % Boolean() - stream async reply? - headers_as_is, % Boolean() - workaround for servers that does - % not honor the http standard, can also be used for testing purposes. - started, % integer() > 0 - When we started processing the request - timer, % undefined | ref() - socket_opts % undefined | [socket_option()] + id, % ref() - Request Id + from, % pid() - Caller + redircount = 0,% Number of redirects made for this request + scheme, % http | https + address, % ({Host,Port}) Destination Host and Port + path, % string() - Path of parsed URL + pquery, % string() - Rest of parsed URL + method, % atom() - HTTP request Method + headers, % #http_request_h{} + content, % {ContentType, Body} - Current HTTP request + settings, % #http_options{} - User defined settings + abs_uri, % string() ex: "http://www.erlang.org" + userinfo, % string() - optinal "<userinfo>@<host>:<port>" + stream, % boolean() - stream async reply? + headers_as_is, % boolean() - workaround for servers that does + % not honor the http standard, can also be used + % for testing purposes. + started, % integer() > 0 - When we started processing the + % request + timer, % undefined | ref() + socket_opts, % undefined | [socket_option()] + ipv6_host_with_brackets % boolean() } ). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index 9015bf1ce2..ab575d867e 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -256,19 +256,27 @@ reset_cookies(ProfileName) -> %%-------------------------------------------------------------------- -%% Function: which_cookies(Url, ProfileName) -> [cookie()] +%% Function: which_cookies(ProfileName) -> [cookie()] +%% which_cookies(Url, ProfileName) -> [cookie()] +%% which_cookies(Url, Options, ProfileName) -> [cookie()] %% %% Url = string() +%% Options = [option()] %% ProfileName = atom() +%% option() = {ipv6_host_with_brackets, boolean()} %% %% Description: Retrieves the cookies that would be sent when %% requesting <Url>. %%-------------------------------------------------------------------- -which_cookies(ProfileName) -> +which_cookies(ProfileName) when is_atom(ProfileName) -> call(ProfileName, which_cookies). -which_cookies(Url, ProfileName) -> - call(ProfileName, {which_cookies, Url}). +which_cookies(Url, ProfileName) + when is_list(Url) andalso is_atom(ProfileName) -> + call(ProfileName, {which_cookies, Url, []}). +which_cookies(Url, Options, ProfileName) + when is_list(Url) andalso is_list(Options) andalso is_atom(ProfileName) -> + call(ProfileName, {which_cookies, Url, Options}). %%-------------------------------------------------------------------- @@ -395,15 +403,16 @@ handle_call(which_cookies, _, #state{cookie_db = CookieDb} = State) -> CookieHeaders = httpc_cookie:which_cookies(CookieDb), {reply, CookieHeaders, State}; -handle_call({which_cookies, Url}, _, #state{cookie_db = CookieDb} = State) -> - ?hcrv("which cookies", [{url, Url}]), - case http_uri:parse(Url) of - {Scheme, _, Host, Port, Path, _} -> +handle_call({which_cookies, Url, Options}, _, + #state{cookie_db = CookieDb} = State) -> + ?hcrv("which cookies", [{url, Url}, {options, Options}]), + case http_uri:parse(Url, Options) of + {ok, {Scheme, _, Host, Port, Path, _}} -> CookieHeaders = httpc_cookie:header(CookieDb, Scheme, {Host, Port}, Path), {reply, CookieHeaders, State}; - Msg -> - {reply, Msg, State} + {error, _} = ERROR -> + {reply, ERROR, State} end; handle_call(info, _, State) -> diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 207b96271c..2414ed0911 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -340,7 +340,9 @@ redirect(Response = {StatusLine, Headers, Body}, Request) -> undefined -> transparent(Response, Request); RedirUrl -> - case http_uri:parse(RedirUrl) of + UrlParseOpts = [{ipv6_host_with_brackets, + Request#request.ipv6_host_with_brackets}], + case http_uri:parse(RedirUrl, UrlParseOpts) of {error, no_scheme} when (Request#request.settings)#http_options.relaxed -> NewLocation = fix_relative_uri(Request, RedirUrl), @@ -350,10 +352,9 @@ redirect(Response = {StatusLine, Headers, Body}, Request) -> {error, Reason} -> {ok, error(Request, Reason), Data}; %% Automatic redirection - {Scheme, _, Host, Port, Path, Query} -> + {ok, {Scheme, _, Host, Port, Path, Query}} -> NewHeaders = - (Request#request.headers)#http_request_h{host = - Host}, + (Request#request.headers)#http_request_h{host = Host}, NewRequest = Request#request{redircount = Request#request.redircount+1, diff --git a/lib/inets/src/http_lib/http_uri.erl b/lib/inets/src/http_lib/http_uri.erl index 44b9face0b..32c6305a79 100644 --- a/lib/inets/src/http_lib/http_uri.erl +++ b/lib/inets/src/http_lib/http_uri.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2010. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,23 +16,30 @@ %% %% %CopyrightEnd% %% -%% +%% +%% RFC 3986 +%% -module(http_uri). --export([parse/1, encode/1, decode/1]). +-export([parse/1, parse/2, + encode/1, decode/1]). + %%%========================================================================= %%% API %%%========================================================================= parse(AbsURI) -> + parse(AbsURI, []). + +parse(AbsURI, Opts) -> case parse_scheme(AbsURI) of {error, Reason} -> {error, Reason}; {Scheme, Rest} -> - case (catch parse_uri_rest(Scheme, Rest)) of + case (catch parse_uri_rest(Scheme, Rest, Opts)) of {UserInfo, Host, Port, Path, Query} -> - {Scheme, UserInfo, Host, Port, Path, Query}; + {ok, {Scheme, UserInfo, Host, Port, Path, Query}}; _ -> {error, {malformed_url, AbsURI}} end @@ -42,35 +49,38 @@ encode(URI) -> Reserved = sets:from_list([$;, $:, $@, $&, $=, $+, $,, $/, $?, $#, $[, $], $<, $>, $\", ${, $}, $|, $\\, $', $^, $%, $ ]), - lists:append(lists:map(fun(Char) -> - uri_encode(Char, Reserved) - end, URI)). - -decode([$%,Hex1,Hex2|Rest]) -> - [hex2dec(Hex1)*16+hex2dec(Hex2)|decode(Rest)]; -decode([First|Rest]) -> - [First|decode(Rest)]; -decode([]) -> + %% lists:append(lists:map(fun(Char) -> uri_encode(Char, Reserved) end, URI)). + lists:append([uri_encode(Char, Reserved) || Char <- URI]). + +decode(String) -> + do_decode(String). + +do_decode([$%,Hex1,Hex2|Rest]) -> + [hex2dec(Hex1)*16+hex2dec(Hex2)|do_decode(Rest)]; +do_decode([First|Rest]) -> + [First|do_decode(Rest)]; +do_decode([]) -> []. + %%%======================================================================== %%% Internal functions %%%======================================================================== + parse_scheme(AbsURI) -> case split_uri(AbsURI, ":", {error, no_scheme}, 1, 1) of {error, no_scheme} -> {error, no_scheme}; {StrScheme, Rest} -> case list_to_atom(http_util:to_lower(StrScheme)) of - Scheme when Scheme == http; Scheme == https -> + Scheme when (Scheme =:= http) orelse (Scheme =:= https) -> {Scheme, Rest}; Scheme -> {error, {not_supported_scheme, Scheme}} end end. -parse_uri_rest(Scheme, "//" ++ URIPart) -> - +parse_uri_rest(Scheme, "//" ++ URIPart, Opts) -> {Authority, PathQuery} = case split_uri(URIPart, "/", URIPart, 1, 0) of Split = {_, _} -> @@ -85,8 +95,8 @@ parse_uri_rest(Scheme, "//" ++ URIPart) -> end, {UserInfo, HostPort} = split_uri(Authority, "@", {"", Authority}, 1, 1), - {Host, Port} = parse_host_port(Scheme, HostPort), - {Path, Query} = parse_path_query(PathQuery), + {Host, Port} = parse_host_port(Scheme, HostPort, Opts), + {Path, Query} = parse_path_query(PathQuery), {UserInfo, Host, Port, Path, Query}. @@ -94,13 +104,14 @@ parse_path_query(PathQuery) -> {Path, Query} = split_uri(PathQuery, "\\?", {PathQuery, ""}, 1, 0), {path(Path), Query}. -parse_host_port(Scheme,"[" ++ HostPort) -> %ipv6 +parse_host_port(Scheme,"[" ++ HostPort, Opts) -> %ipv6 DefaultPort = default_port(Scheme), {Host, ColonPort} = split_uri(HostPort, "\\]", {HostPort, ""}, 1, 1), + Host2 = maybe_ipv6_host_with_brackets(Host, Opts), {_, Port} = split_uri(ColonPort, ":", {"", DefaultPort}, 0, 1), - {Host, int_port(Port)}; + {Host2, int_port(Port)}; -parse_host_port(Scheme, HostPort) -> +parse_host_port(Scheme, HostPort, _Opts) -> DefaultPort = default_port(Scheme), {Host, Port} = split_uri(HostPort, ":", {HostPort, DefaultPort}, 1, 1), {Host, int_port(Port)}. @@ -114,6 +125,14 @@ split_uri(UriPart, SplitChar, NoMatchResult, SkipLeft, SkipRight) -> NoMatchResult end. +maybe_ipv6_host_with_brackets(Host, Opts) -> + case lists:keysearch(ipv6_host_with_brackets, 1, Opts) of + {value, {ipv6_host_with_brackets, true}} -> + "[" ++ Host ++ "]"; + _ -> + Host + end. + default_port(http) -> 80; default_port(https) -> diff --git a/lib/inets/src/http_lib/http_util.erl b/lib/inets/src/http_lib/http_util.erl index 973600d7be..5b21170b78 100644 --- a/lib/inets/src/http_lib/http_util.erl +++ b/lib/inets/src/http_lib/http_util.erl @@ -206,9 +206,7 @@ timeout(Timeout, Started) -> html_encode(Chars) -> Reserved = sets:from_list([$&, $<, $>, $\", $', $/]), - lists:append(lists:map(fun(Char) -> - char_to_html_entity(Char, Reserved) - end, Chars)). + lists:append([char_to_html_entity(Char, Reserved) || Char <- Chars]). %%%======================================================================== diff --git a/lib/inets/src/http_server/httpd_file.erl b/lib/inets/src/http_server/httpd_file.erl index e8a8ab6411..f2ba33099e 100644 --- a/lib/inets/src/http_server/httpd_file.erl +++ b/lib/inets/src/http_server/httpd_file.erl @@ -36,9 +36,9 @@ handle_error(emfile, Op, _ModData, Path) -> handle_error(500, Op, none, Path, ": Too many open files"); handle_error({enfile,_}, Op, _ModData, Path) -> handle_error(500, Op, none, Path, ": File table overflow"); -handle_error(_Reason, Op, ModData, Path) -> - handle_error(404, Op, ModData, Path, ": File not found"). - +handle_error(_Reason, Op, _ModData, Path) -> + handle_error(500, Op, none, Path, ""). + handle_error(StatusCode, Op, none, Path, Reason) -> {StatusCode, none, ?NICE("Can't " ++ Op ++ " " ++ Path ++ Reason)}; handle_error(StatusCode, Op, ModData, Path, Reason) -> diff --git a/lib/inets/src/http_server/httpd_request.erl b/lib/inets/src/http_server/httpd_request.erl index a04bcc2778..5ba79b2706 100644 --- a/lib/inets/src/http_server/httpd_request.erl +++ b/lib/inets/src/http_server/httpd_request.erl @@ -309,12 +309,12 @@ validate_uri(RequestURI) -> (catch http_uri:decode(string:left(RequestURI, Ndx))) end, case UriNoQueryNoHex of - {'EXIT',_Reason} -> + {'EXIT', _Reason} -> {error, {bad_request, {malformed_syntax, RequestURI}}}; _ -> Path = format_request_uri(UriNoQueryNoHex), - Path2 = [X||X<-string:tokens(Path, "/\\"),X=/="."], - validate_path( Path2,0, RequestURI) + Path2 = [X||X<-string:tokens(Path, "/"),X=/="."], %% OTP-5938 + validate_path(Path2, 0, RequestURI) end. validate_path([], _, _) -> diff --git a/lib/inets/src/http_server/httpd_request_handler.erl b/lib/inets/src/http_server/httpd_request_handler.erl index c3b47ce390..d2f22fce93 100644 --- a/lib/inets/src/http_server/httpd_request_handler.erl +++ b/lib/inets/src/http_server/httpd_request_handler.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 1997-2011. All Rights Reserved. -%% +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the @@ -355,7 +355,7 @@ handle_http_msg({Method, Uri, Version, {RecordHeaders, Headers}, Body}, Reason = io_lib:format("Forbidden URI: ~p~n", [URI]), error_log(Reason, ModData), {stop, normal, State#state{response_sent = true}}; - {error,{bad_request, {malformed_syntax, URI}}} -> + {error, {bad_request, {malformed_syntax, URI}}} -> ?hdrd("validation failed: bad request - malformed syntax", [{uri, URI}]), httpd_response:send_status(ModData#mod{http_version = Version}, diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index ea9cfbf4f2..dd7223876e 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -78,6 +78,7 @@ traverse_modules(ModData,[Module|Rest]) -> [Module, Reason])), report_error(mod_log, ModData#mod.config_db, String), report_error(mod_disk_log, ModData#mod.config_db, String), + send_status(ModData, 500, none), done; done -> ?hdrt("traverse modules - done", []), @@ -100,12 +101,19 @@ send_status(#mod{socket_type = SocketType, socket = Socket, config_db = ConfigDB} = ModData, StatusCode, PhraseArgs) -> + ?hdrd("send status", [{status_code, StatusCode}, + {phrase_args, PhraseArgs}]), + ReasonPhrase = httpd_util:reason_phrase(StatusCode), Message = httpd_util:message(StatusCode, PhraseArgs, ConfigDB), Body = get_body(ReasonPhrase, Message), - send_header(ModData, StatusCode, [{content_type, "text/html"}, - {content_length, integer_to_list(length(Body))}]), + ?hdrt("send status - header", [{reason_phrase, ReasonPhrase}, + {message, Message}]), + send_header(ModData, StatusCode, + [{content_type, "text/html"}, + {content_length, integer_to_list(length(Body))}]), + httpd_socket:deliver(SocketType, Socket, Body). @@ -345,8 +353,9 @@ transform({Field, Value}) when is_list(Field) -> %% Leave this method and go on to the newer form of response %% OTP-4408 %%---------------------------------------------------------------------- -send_response_old(#mod{method = "HEAD"} = ModData, +send_response_old(#mod{method = "HEAD"} = ModData, StatusCode, Response) -> + NewResponse = lists:flatten(Response), case httpd_util:split(NewResponse, [?CR, ?LF, ?CR, ?LF],2) of diff --git a/lib/inets/src/http_server/httpd_util.erl b/lib/inets/src/http_server/httpd_util.erl index c051422529..b0b18b9c3d 100644 --- a/lib/inets/src/http_server/httpd_util.erl +++ b/lib/inets/src/http_server/httpd_util.erl @@ -178,11 +178,12 @@ message(301,URL,_) -> "The document has moved <A HREF=\""++ maybe_encode(URL) ++"\">here</A>."; message(304, _URL,_) -> "The document has not been changed."; -message(400,none,_) -> - "Your browser sent a query that this server could not understand."; -message(400,Msg,_) -> - "Your browser sent a query that this server could not understand. "++ http_util:html_encode(Msg); -message(401,none,_) -> +message(400, none, _) -> + "Your browser sent a query that this server could not understand. "; +message(400, Msg, _) -> + "Your browser sent a query that this server could not understand. " ++ + html_encode(Msg); +message(401, none, _) -> "This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong @@ -190,40 +191,49 @@ credentials (e.g., bad password), or your browser doesn't understand how to supply the credentials required."; message(403,RequestURI,_) -> - "You don't have permission to access "++ http_util:html_encode(RequestURI) ++" on this server."; + "You don't have permission to access " ++ + html_encode(RequestURI) ++ + " on this server."; message(404,RequestURI,_) -> - "The requested URL " ++ http_util:html_encode(RequestURI) ++ " was not found on this server."; + "The requested URL " ++ + html_encode(RequestURI) ++ + " was not found on this server."; message(408, Timeout, _) -> Timeout; message(412,none,_) -> "The requested preconditions were false"; message(413, Reason,_) -> - "Entity: " ++ http_util:html_encode(Reason); + "Entity: " ++ html_encode(Reason); message(414,ReasonPhrase,_) -> - "Message "++ http_util:html_encode(ReasonPhrase) ++"."; + "Message " ++ html_encode(ReasonPhrase) ++ "."; message(416,ReasonPhrase,_) -> - http_util:html_encode(ReasonPhrase); + html_encode(ReasonPhrase); message(500,_,ConfigDB) -> ServerAdmin=lookup(ConfigDB,server_admin,"unknown@unknown"), "The server encountered an internal error or " "misconfiguration and was unable to complete " "your request.<P>Please contact the server administrator " - ++ http_util:html_encode(ServerAdmin) ++ ", and inform them of the time the error occurred " + ++ html_encode(ServerAdmin) ++ + ", and inform them of the time the error occurred " "and anything you might have done that may have caused the error."; message(501,{Method, RequestURI, HTTPVersion}, _ConfigDB) -> if is_atom(Method) -> - http_util:html_encode(atom_to_list(Method))++ - " to "++ http_util:html_encode(RequestURI)++" ("++ http_util:html_encode(HTTPVersion)++") not supported."; + atom_to_list(Method) ++ + " to " ++ + html_encode(RequestURI) ++ + " (" ++ HTTPVersion ++ ") not supported."; is_list(Method) -> - http_util:html_encode(Method)++ - " to "++ http_util:html_encode(RequestURI)++" ("++ http_util:html_encode(HTTPVersion)++") not supported." + Method ++ + " to " ++ + html_encode(RequestURI) ++ + " (" ++ HTTPVersion ++ ") not supported." end; message(503, String, _ConfigDB) -> - "This service in unavailable due to: "++ http_util:html_encode(String). + "This service in unavailable due to: " ++ html_encode(String). maybe_encode(URI) -> Decoded = try http_uri:decode(URI) of @@ -233,6 +243,15 @@ maybe_encode(URI) -> end, http_uri:encode(Decoded). +html_encode(String) -> + try http_uri:decode(String) of + Decoded when is_list(Decoded) -> + http_util:html_encode(Decoded) + catch + _:_ -> + http_util:html_encode(String) + end. + %%convert_rfc_date(Date)->{{YYYY,MM,DD},{HH,MIN,SEC}} convert_request_date([D,A,Y,DateType| Rest])-> @@ -245,7 +264,7 @@ convert_request_date([D,A,Y,DateType| Rest])-> fun convert_rfc850_date/1 end, case catch Func([D,A,Y,DateType| Rest]) of - {ok,Date} -> + {ok, Date} -> Date; _Error-> bad_date diff --git a/lib/inets/src/http_server/mod_responsecontrol.erl b/lib/inets/src/http_server/mod_responsecontrol.erl index 5d5b60cdbd..989f45db20 100644 --- a/lib/inets/src/http_server/mod_responsecontrol.erl +++ b/lib/inets/src/http_server/mod_responsecontrol.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2010. All Rights Reserved. +%% Copyright Ericsson AB 2001-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -209,14 +209,14 @@ compare_etags(Tag,Etags) -> nomatch end. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%%Control if the file is modificated %% -%% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% %% +%% Control if the file is modificated %% +%% %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%---------------------------------------------------------------------- -%%Control the If-Modified-Since and If-Not-Modified-Since header fields +%% Control the If-Modified-Since and If-Not-Modified-Since header fields %%---------------------------------------------------------------------- control_modification(Path,Info,FileInfo)-> ?DEBUG("control_modification() -> entry",[]), @@ -227,6 +227,8 @@ control_modification(Path,Info,FileInfo)-> continue; unmodified-> {304, Info, Path}; + {bad_date, _} = BadDate-> + {400, Info, BadDate}; undefined -> case control_modification_data(Info, FileInfo#file_info.mtime, @@ -253,21 +255,27 @@ control_modification_data(Info, ModificationTime, HeaderField)-> undefined-> undefined; LastModified0 -> - LastModified = calendar:universal_time_to_local_time( - httpd_util:convert_request_date(LastModified0)), - ?DEBUG("control_modification_data() -> " - "~n Request-Field: ~s" - "~n FileLastModified: ~p" - "~n FieldValue: ~p", - [HeaderField, ModificationTime, LastModified]), - FileTime = - calendar:datetime_to_gregorian_seconds(ModificationTime), - FieldTime = calendar:datetime_to_gregorian_seconds(LastModified), - if - FileTime =< FieldTime -> - ?DEBUG("File unmodified~n", []), unmodified; - FileTime >= FieldTime -> - ?DEBUG("File modified~n", []), modified + case httpd_util:convert_request_date(LastModified0) of + bad_date -> + {bad_date, LastModified0}; + ConvertedReqDate -> + LastModified = + calendar:universal_time_to_local_time(ConvertedReqDate), + ?DEBUG("control_modification_data() -> " + "~n Request-Field: ~s" + "~n FileLastModified: ~p" + "~n FieldValue: ~p", + [HeaderField, ModificationTime, LastModified]), + FileTime = + calendar:datetime_to_gregorian_seconds(ModificationTime), + FieldTime = + calendar:datetime_to_gregorian_seconds(LastModified), + if + FileTime =< FieldTime -> + ?DEBUG("File unmodified~n", []), unmodified; + FileTime >= FieldTime -> + ?DEBUG("File modified~n", []), modified + end end end. @@ -285,6 +293,9 @@ strip_date([C | Rest]) -> send_return_value({412,_,_}, _FileInfo)-> {status,{412,none,"Precondition Failed"}}; +send_return_value({400,_, {bad_date, BadDate}}, _FileInfo)-> + {status, {400, none, "Bad date: " ++ BadDate}}; + send_return_value({304,Info,Path}, FileInfo)-> Suffix = httpd_util:suffix(Path), MimeType = httpd_util:lookup_mime_default(Info#mod.config_db,Suffix, diff --git a/lib/inets/src/inets_app/Makefile b/lib/inets/src/inets_app/Makefile index 20e22917e2..63ab0a5bae 100644 --- a/lib/inets/src/inets_app/Makefile +++ b/lib/inets/src/inets_app/Makefile @@ -30,6 +30,7 @@ include ../../vsn.mk VSN = $(INETS_VSN) + # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- @@ -41,8 +42,8 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # ---------------------------------------------------- MODULES = \ - inets_service \ inets \ + inets_service \ inets_app \ inets_sup \ inets_regexp diff --git a/lib/inets/src/inets_app/inets.appup.src b/lib/inets/src/inets_app/inets.appup.src index e6d315819c..779dd8e439 100644 --- a/lib/inets/src/inets_app/inets.appup.src +++ b/lib/inets/src/inets_app/inets.appup.src @@ -18,6 +18,11 @@ {"%VSN%", [ + {"5.7.2", + [ + {restart_application, inets} + ] + }, {"5.7.1", [ {restart_application, inets} @@ -35,6 +40,11 @@ } ], [ + {"5.7.2", + [ + {restart_application, inets} + ] + }, {"5.7.1", [ {restart_application, inets} diff --git a/lib/inets/src/inets_app/inets.mk b/lib/inets/src/inets_app/inets.mk index b6e9fe1d96..35fb0d7eca 100644 --- a/lib/inets/src/inets_app/inets.mk +++ b/lib/inets/src/inets_app/inets.mk @@ -33,6 +33,10 @@ ifeq ($(WARN_UNUSED_WARS), true) ERL_COMPILE_FLAGS += +warn_unused_vars endif +ifeq ($(shell erl -noshell -eval 'io:format("~4s", [erlang:system_info(otp_release)])' -s init stop), R14B) +INETS_ERL_COMPILE_FLAGS += -D'OTP-R14B-COMPILER' +endif + INETS_APP_VSN_COMPILE_FLAGS = \ +'{parse_transform,sys_pre_attributes}' \ +'{attribute,insert,app_vsn,$(APP_VSN)}' diff --git a/lib/inets/src/inets_app/inets_service.erl b/lib/inets/src/inets_app/inets_service.erl index f89dac195c..a057a51e2c 100644 --- a/lib/inets/src/inets_app/inets_service.erl +++ b/lib/inets/src/inets_app/inets_service.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,6 +20,21 @@ -module(inets_service). +-ifdef('OTP-R14B-COMPILER'). + +-export([behaviour_info/1]). + +behaviour_info(callbacks) -> + [{start_standalone, 1}, + {start_service, 1}, + {stop_service, 1}, + {services, 0}, + {service_info, 1}]; +behaviour_info(_) -> + undefined. + +-else. + %% Starts service stand-alone %% start_standalone(Config) -> % {ok, Pid} | {error, Reason} %% <service>:start_link(Config). @@ -67,3 +82,5 @@ -callback service_info(Service :: term()) -> {ok, [{Property :: term(), Value :: term()}]} | {error, Reason :: term()}. + +-endif. diff --git a/lib/inets/src/tftp/tftp.erl b/lib/inets/src/tftp/tftp.erl index b33c0a98f4..8d8fce388d 100644 --- a/lib/inets/src/tftp/tftp.erl +++ b/lib/inets/src/tftp/tftp.erl @@ -224,6 +224,20 @@ service_info/1 ]). +-ifdef('OTP-R14B-COMPILER'). + +-export([behaviour_info/1]). + +behaviour_info(callbacks) -> + [{prepare, 6}, + {open, 6}, + {read, 1}, + {write, 2}, + {abort, 3}]; +behaviour_info(_) -> + undefined. + +-else. -type peer() :: {PeerType :: inet | inet6, PeerHost :: inet:ip_address(), @@ -266,6 +280,8 @@ -callback abort(Code :: error_code(), string(), State :: term()) -> 'ok'. +-endif. + -include("tftp.hrl"). diff --git a/lib/inets/test/ftp_suite_lib.erl b/lib/inets/test/ftp_suite_lib.erl index 3ebd02229e..ffb58c91b6 100644 --- a/lib/inets/test/ftp_suite_lib.erl +++ b/lib/inets/test/ftp_suite_lib.erl @@ -196,7 +196,9 @@ test_filenames() -> %% variable, but should NOT alter/remove any existing entries. %%-------------------------------------------------------------------- init_per_testcase(Case, Config) - when (Case =:= open) orelse (Case =:= open_port) -> + when (Case =:= open) orelse + (Case =:= open_port) -> + put(ftp_testcase, Case), io:format(user, "~n~n*** INIT ~w:~w ***~n~n", [?MODULE, Case]), inets:start(), NewConfig = data_dir(Config), @@ -266,7 +268,7 @@ do_init_per_testcase(Case, Config) -> end, Opts2 = case string:tokens(atom_to_list(Case), [$_]) of - [_, "active" | _] -> + ["active" | _] -> [{mode, active} | Opts1]; _ -> [{mode, passive} | Opts1] @@ -367,8 +369,11 @@ open(Config) when is_list(Config) -> tc_open(Host) -> + p("tc_open -> entry with" + "~n Host: ~p", [Host]), {ok, Pid} = ?ftp_open(Host, []), ok = ftp:close(Pid), + p("tc_open -> try (ok) open 1"), {ok, Pid1} = ftp:open({option_list, [{host,Host}, {port, ?FTP_PORT}, @@ -376,11 +381,13 @@ tc_open(Host) -> {timeout, 30000}]}), ok = ftp:close(Pid1), + p("tc_open -> try (fail) open 2"), {error, ehost} = ftp:open({option_list, [{port, ?FTP_PORT}, {flags, [verbose]}]}), {ok, Pid2} = ftp:open(Host), ok = ftp:close(Pid2), + p("tc_open -> try (ok) open 3"), {ok, NewHost} = inet:getaddr(Host, inet), {ok, Pid3} = ftp:open(NewHost), ftp:user(Pid3, ?FTP_USER, ?FTP_PASS), @@ -392,33 +399,68 @@ tc_open(Host) -> %% Bad input that has default values are ignored and the defult %% is used. + p("tc_open -> try (ok) open 4"), {ok, Pid4} = - ftp:open({option_list, [{host, Host}, {port, badarg}, - {flags, [verbose]}, + ftp:open({option_list, [{host, Host}, + {port, badarg}, + {flags, [verbose]}, {timeout, 30000}]}), test_server:sleep(100), ok = ftp:close(Pid4), + + p("tc_open -> try (ok) open 5"), {ok, Pid5} = - ftp:open({option_list, [{host, Host}, {port, ?FTP_PORT}, - {flags, [verbose]}, + ftp:open({option_list, [{host, Host}, + {port, ?FTP_PORT}, + {flags, [verbose]}, {timeout, -42}]}), test_server:sleep(100), ok = ftp:close(Pid5), + + p("tc_open -> try (ok) open 6"), {ok, Pid6} = - ftp:open({option_list, [{host, Host}, {port, ?FTP_PORT}, + ftp:open({option_list, [{host, Host}, + {port, ?FTP_PORT}, {flags, [verbose]}, - {mode, cool}]}), + {mode, cool}]}), test_server:sleep(100), ok = ftp:close(Pid6), + p("tc_open -> try (ok) open 7"), {ok, Pid7} = ftp:open(Host, [{port, ?FTP_PORT}, {verbose, true}, {timeout, 30000}]), ok = ftp:close(Pid7), + p("tc_open -> try (ok) open 8"), {ok, Pid8} = ftp:open(Host, ?FTP_PORT), ok = ftp:close(Pid8), + p("tc_open -> try (ok) open 9"), + {ok, Pid9} = + ftp:open(Host, [{port, ?FTP_PORT}, + {verbose, true}, + {timeout, 30000}, + {dtimeout, -99}]), + ok = ftp:close(Pid9), + + p("tc_open -> try (ok) open 10"), + {ok, Pid10} = + ftp:open(Host, [{port, ?FTP_PORT}, + {verbose, true}, + {timeout, 30000}, + {dtimeout, "foobar"}]), + ok = ftp:close(Pid10), + + p("tc_open -> try (ok) open 11"), + {ok, Pid11} = + ftp:open(Host, [{port, ?FTP_PORT}, + {verbose, true}, + {timeout, 30000}, + {dtimeout, 1}]), + ok = ftp:close(Pid11), + + p("tc_open -> done"), ok. @@ -445,7 +487,7 @@ passive_user(suite) -> []; passive_user(Config) when is_list(Config) -> Pid = ?config(ftp, Config), - io:format("Pid: ~p~n",[Pid]), + p("Pid: ~p",[Pid]), do_user(Pid). @@ -967,13 +1009,13 @@ api_missuse(doc)-> ["Test that behaviour of the ftp process if the api is abused"]; api_missuse(suite) -> []; api_missuse(Config) when is_list(Config) -> - io:format("api_missuse -> entry~n", []), + p("api_missuse -> entry"), Flag = process_flag(trap_exit, true), Pid = ?config(ftp, Config), Host = ftp_host(Config), %% Serious programming fault, connetion will be shut down - io:format("api_missuse -> verify bad call termination (~p)~n", [Pid]), + p("api_missuse -> verify bad call termination (~p)", [Pid]), case (catch gen_server:call(Pid, {self(), foobar, 10}, infinity)) of {error, {connection_terminated, 'API_violation'}} -> ok; @@ -983,23 +1025,23 @@ api_missuse(Config) when is_list(Config) -> test_server:sleep(500), undefined = process_info(Pid, status), - io:format("api_missuse -> start new client~n", []), + p("api_missuse -> start new client"), {ok, Pid2} = ?ftp_open(Host, []), %% Serious programming fault, connetion will be shut down - io:format("api_missuse -> verify bad cast termination~n", []), + p("api_missuse -> verify bad cast termination"), gen_server:cast(Pid2, {self(), foobar, 10}), test_server:sleep(500), undefined = process_info(Pid2, status), - io:format("api_missuse -> start new client~n", []), + p("api_missuse -> start new client"), {ok, Pid3} = ?ftp_open(Host, []), %% Could be an innocent misstake the connection lives. - io:format("api_missuse -> verify bad bang~n", []), + p("api_missuse -> verify bad bang"), Pid3 ! foobar, test_server:sleep(500), {status, _} = process_info(Pid3, status), process_flag(trap_exit, Flag), - io:format("api_missuse -> done~n", []), + p("api_missuse -> done"), ok. @@ -1567,9 +1609,9 @@ split([], I, Is) -> lists:reverse([lists:reverse(I)| Is]). do_ftp_open(Host, Opts) -> - io:format("do_ftp_open -> entry with" - "~n Host: ~p" - "~n Opts: ~p", [Host, Opts]), + p("do_ftp_open -> entry with" + "~n Host: ~p" + "~n Opts: ~p", [Host, Opts]), case ftp:open(Host, Opts) of {ok, _} = OK -> OK; @@ -1595,7 +1637,7 @@ passwd() -> ftpd_hosts(Config) -> DataDir = ?config(data_dir, Config), FileName = filename:join([DataDir, "../ftp_SUITE_data/", ftpd_hosts]), - io:format("FileName: ~p~n", [FileName]), + p("FileName: ~p", [FileName]), case file:consult(FileName) of {ok, [Hosts]} when is_list(Hosts) -> Hosts; diff --git a/lib/inets/test/ftp_windows_2003_server_test.erl b/lib/inets/test/ftp_windows_2003_server_test.erl index 57f1ae8358..32f25713f8 100644 --- a/lib/inets/test/ftp_windows_2003_server_test.erl +++ b/lib/inets/test/ftp_windows_2003_server_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -87,14 +87,22 @@ end_per_testcase(Case, Config) -> %% Description: Returns a list of all test cases in this test suite %%-------------------------------------------------------------------- all() -> - [open, open_port, {group, passive}, {group, active}, - api_missuse, not_owner, {group, progress_report}]. + [ + open, + open_port, + {group, passive}, + {group, active}, + api_missuse, + not_owner, + {group, progress_report} + ]. groups() -> - [{passive, [], ftp_suite_lib:passive(suite)}, - {active, [], ftp_suite_lib:active(suite)}, - {progress_report, [], - ftp_suite_lib:progress_report(suite)}]. + [ + {passive, [], ftp_suite_lib:passive(suite)}, + {active, [], ftp_suite_lib:active(suite)}, + {progress_report, [], ftp_suite_lib:progress_report(suite)} + ]. init_per_group(_GroupName, Config) -> Config. diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index f95fb93669..d86e52f3d5 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -250,10 +250,10 @@ init_per_testcase(Case, Config) -> init_per_testcase(Case, 2, Config). init_per_testcase(Case, Timeout, Config) -> - io:format(user, "~n~n*** INIT ~w:~w[~w] ***~n~n", - [?MODULE, Case, Timeout]), - PrivDir = ?config(priv_dir, Config), - tsp("init_per_testcase -> stop inets"), + io:format(user, + "~n~n*** INIT ~w:~w[~w] ***" + "~n~n", [?MODULE, Case, Timeout]), + PrivDir = ?config(priv_dir, Config), application:stop(inets), Dog = test_server:timetrap(inets_test_lib:minutes(Timeout)), TmpConfig = lists:keydelete(watchdog, 1, Config), @@ -289,12 +289,12 @@ init_per_testcase(Case, Timeout, Config) -> throw:{error, {failed_starting, App, _}} -> SkipString = "Could not start " ++ atom_to_list(App), - {skip, SkipString}; - _:X -> + skip(SkipString); + _:X -> SkipString = lists:flatten( io_lib:format("Failed starting apps: ~p", [X])), - {skip, SkipString} + skip(SkipString) end; _ -> @@ -323,14 +323,14 @@ init_per_testcase(Case, Timeout, Config) -> ], case lists:member(Rest, BadCases) of true -> - [{skip, "TC and server not compatible"}| + [skip("TC and server not compatible") | TmpConfig]; false -> inets:start(), [{watchdog, Dog} | TmpConfig] end; false -> - [{skip, "proxy not responding"} | TmpConfig] + [skip("proxy not responding") | TmpConfig] end end; @@ -360,20 +360,19 @@ init_per_testcase(Case, Timeout, Config) -> io_lib:format("Failed starting apps: ~p", [X])), {skip, SkipString} end; + _ -> TmpConfig2 = lists:keydelete(local_server, 1, TmpConfig), - Server = - %% Will start inets - inets_test_lib:start_http_server( - filename:join(PrivDir, IpConfFile)), + %% Will start inets + Server = start_http_server(PrivDir, IpConfFile), [{watchdog, Dog}, {local_server, Server} | TmpConfig2] end, %% This will fail for the ipv6_ - cases (but that is ok) - httpc:set_options([{proxy, {{?PROXY, ?PROXY_PORT}, - ["localhost", ?IPV6_LOCAL_HOST]}}, - {ipfamily, inet6fb4}]), - + ProxyExceptions = ["localhost", ?IPV6_LOCAL_HOST], + http:set_options([{proxy, {{?PROXY, ?PROXY_PORT}, ProxyExceptions}}]), + inets:enable_trace(max, io, httpc), + %% inets:enable_trace(max, io, all), %% snmp:set_trace([gen_tcp]), NewConfig. @@ -390,7 +389,10 @@ init_per_testcase_ssl(Tag, PrivDir, SslConfFile, Config) -> tsp("init_per_testcase(~w) -> Server: ~p", [Tag, Server]), [{local_ssl_server, Server} | Config2]. - +start_http_server(ConfDir, ConfFile) -> + inets_test_lib:start_http_server( filename:join(ConfDir, ConfFile) ). + + %%-------------------------------------------------------------------- %% Function: end_per_testcase(Case, Config) -> _ %% Case - atom() @@ -726,7 +728,7 @@ test_pipeline(URL) -> p("test_pipeline -> received reply for (async) request 2"), ok; {http, Msg1} -> - test_server:fail(Msg1) + tsf(Msg1) end; {http, {RequestId2, {{_, 200, _}, _, _}}} -> io:format("test_pipeline -> received reply for (async) request 2 - now wait for 1"), @@ -735,14 +737,14 @@ test_pipeline(URL) -> io:format("test_pipeline -> received reply for (async) request 1"), ok; {http, Msg2} -> - test_server:fail(Msg2) + tsf(Msg2) end; {http, Msg3} -> - test_server:fail(Msg3) + tsf(Msg3) after 60000 -> receive Any1 -> tsp("received crap after timeout: ~n ~p", [Any1]), - test_server:fail({error, {timeout, Any1}}) + tsf({error, {timeout, Any1}}) end end, @@ -767,7 +769,7 @@ test_pipeline(URL) -> p("test_pipeline -> expect *no* reply for cancelled (async) request 4 (for 3 secs)"), receive {http, {RequestId3, _}} -> - test_server:fail(http_cancel_request_failed) + tsf(http_cancel_request_failed) after 3000 -> ok end, @@ -780,11 +782,11 @@ test_pipeline(URL) -> tsp("Receive : ~p", [Res]), BinBody4; {http, Msg4} -> - test_server:fail(Msg4) + tsf(Msg4) after 60000 -> receive Any2 -> tsp("received crap after timeout: ~n ~p", [Any2]), - test_server:fail({error, {timeout, Any2}}) + tsf({error, {timeout, Any2}}) end end, @@ -794,7 +796,7 @@ test_pipeline(URL) -> p("test_pipeline -> ensure no unexpected incomming"), receive {http, Any} -> - test_server:fail({unexpected_message, Any}) + tsf({unexpected_message, Any}) after 500 -> ok end, @@ -816,11 +818,11 @@ http_trace(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], "TRACE /dummy.html" ++ _}} -> ok; {ok, {{_,200,_}, [_ | _], WrongBody}} -> - test_server:fail({wrong_body, WrongBody}); + tsf({wrong_body, WrongBody}); {ok, WrongReply} -> - test_server:fail({wrong_reply, WrongReply}); + tsf({wrong_reply, WrongReply}); Error -> - test_server:fail({failed, Error}) + tsf({failed, Error}) end; _ -> {skip, "Failed to start local http-server"} @@ -843,7 +845,7 @@ http_async(Config) when is_list(Config) -> {http, {RequestId, {{_, 200, _}, _, BinBody}}} -> BinBody; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, inets_test_lib:check_body(binary_to_list(Body)), @@ -853,7 +855,7 @@ http_async(Config) when is_list(Config) -> ok = httpc:cancel_request(NewRequestId), receive {http, {NewRequestId, _NewResult}} -> - test_server:fail(http_cancel_request_failed) + tsf(http_cancel_request_failed) after 3000 -> ok end; @@ -902,7 +904,7 @@ http_save_to_file_async(Config) when is_list(Config) -> {http, {RequestId, saved_to_file}} -> ok; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, {ok, Bin} = file:read_file(FilePath), @@ -1448,10 +1450,10 @@ proxy_options(Config) when is_list(Config) -> {value, {"allow", _}} -> ok; _ -> - test_server:fail(http_options_request_failed) + tsf(http_options_request_failed) end; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1472,7 +1474,7 @@ proxy_head(Config) when is_list(Config) -> {ok, {{_,200, _}, [_ | _], []}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1491,7 +1493,7 @@ proxy_get(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} -> inets_test_lib:check_body(Body); Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1570,7 +1572,7 @@ proxy_post(Config) when is_list(Config) -> {ok, {{_,405,_}, [_ | _], [_ | _]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1595,7 +1597,7 @@ proxy_put(Config) when is_list(Config) -> {ok, {{_,405,_}, [_ | _], [_ | _]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1620,7 +1622,7 @@ proxy_delete(Config) when is_list(Config) -> {ok, {{_,404,_}, [_ | _], [_ | _]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1676,7 +1678,7 @@ proxy_auth(Config) when is_list(Config) -> {ok, {{_,200, _}, [_ | _], [_|_]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1762,7 +1764,7 @@ http_stream(Config) when is_list(Config) -> {http, {RequestId, stream_start, _Headers}} -> ok; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, StreamedBody = receive_streamed_body(RequestId, <<>>), @@ -1817,7 +1819,7 @@ once(URL) -> [RequestId, Pid]), Pid; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, tsp("once -> request handler: ~p", [NewPid]), @@ -1860,7 +1862,7 @@ proxy_stream(Config) when is_list(Config) -> {http, {RequestId, stream_start, _Headers}} -> ok; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, StreamedBody = receive_streamed_body(RequestId, <<>>), @@ -1878,22 +1880,31 @@ parse_url(suite) -> []; parse_url(Config) when is_list(Config) -> %% ipv6 - {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]} - = http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html"), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html"), + {ok, {http,[],"[2010:836B:4179::836B:4179]",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{ipv6_host_with_brackets, true}]), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{ipv6_host_with_brackets, false}]), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{foo, false}]), {error, {malformed_url,"http://2010:836B:4179::836B:4179/foobar.html"}} = http_uri:parse("http://2010:836B:4179::836B:4179/foobar.html"), %% ipv4 - {http,[],"127.0.0.1",80,"/foobar.html",[]} = + {ok, {http,[],"127.0.0.1",80,"/foobar.html",[]}} = http_uri:parse("http://127.0.0.1/foobar.html"), %% host - {http,[],"localhost",8888,"/foobar.html",[]} = + {ok, {http,[],"localhost",8888,"/foobar.html",[]}} = http_uri:parse("http://localhost:8888/foobar.html"), %% Userinfo - {http,"nisse:foobar","localhost",8888,"/foobar.html",[]} = + {ok, {http,"nisse:foobar","localhost",8888,"/foobar.html",[]}} = http_uri:parse("http://nisse:foobar@localhost:8888/foobar.html"), %% Scheme error @@ -1902,18 +1913,20 @@ parse_url(Config) when is_list(Config) -> http_uri:parse("localhost:8888/foobar.html"), %% Query - {http,[],"localhost",8888,"/foobar.html","?foo=bar&foobar=42"} = + {ok, {http,[],"localhost",8888,"/foobar.html","?foo=bar&foobar=42"}} = http_uri:parse("http://localhost:8888/foobar.html?foo=bar&foobar=42"), %% Esc chars - {http,[],"www.somedomain.com",80,"/%2Eabc",[]} = + {ok, {http,[],"www.somedomain.com",80,"/%2Eabc",[]}} = http_uri:parse("http://www.somedomain.com/%2Eabc"), - {http,[],"www.somedomain.com",80,"/%252Eabc",[]} = + {ok, {http,[],"www.somedomain.com",80,"/%252Eabc",[]}} = http_uri:parse("http://www.somedomain.com/%252Eabc"), - {http,[],"www.somedomain.com",80,"/%25abc",[]} = + {ok, {http,[],"www.somedomain.com",80,"/%25abc",[]}} = http_uri:parse("http://www.somedomain.com/%25abc"), - {http,[],"www.somedomain.com",80,"/%25abc", "?foo=bar"} = + {ok, {http,[],"www.somedomain.com",80,"/%25abc", "?foo=bar"}} = http_uri:parse("http://www.somedomain.com/%25abc?foo=bar"), + + ok. @@ -2058,12 +2071,14 @@ http_invalid_http(Config) when is_list(Config) -> %%------------------------------------------------------------------------- +-define(GOOGLE, "www.google.com"). + hexed_query_otp_6191(doc) -> []; hexed_query_otp_6191(suite) -> []; hexed_query_otp_6191(Config) when is_list(Config) -> - Google = "www.google.com", + Google = ?GOOGLE, GoogleSearch = "http://" ++ Google ++ "/search", Search1 = "?hl=en&q=a%D1%85%D1%83%D0%B9&btnG=Google+Search", URI1 = GoogleSearch ++ Search1, @@ -2072,11 +2087,32 @@ hexed_query_otp_6191(Config) when is_list(Config) -> Search3 = "?hl=en&q=%foo", URI3 = GoogleSearch ++ Search3, - {http, [], Google, 80, "/search", _} = http_uri:parse(URI1), - {http, [], Google, 80, "/search", _} = http_uri:parse(URI2), - {http, [], Google, 80, "/search", _} = http_uri:parse(URI3), + Verify1 = + fun({http, [], ?GOOGLE, 80, "/search", _}) -> ok; + (_) -> error + end, + Verify2 = Verify1, + Verify3 = Verify1, + verify_uri(URI1, Verify1), + verify_uri(URI2, Verify2), + verify_uri(URI3, Verify3), ok. +verify_uri(URI, Verify) -> + case http_uri:parse(URI) of + {ok, ParsedURI} -> + case Verify(ParsedURI) of + ok -> + ok; + error -> + Reason = {unexpected_parse_result, URI, ParsedURI}, + ERROR = {error, Reason}, + throw(ERROR) + end; + {error, _} = ERROR -> + throw(ERROR) + end. + %%------------------------------------------------------------------------- @@ -2945,7 +2981,7 @@ receive_streamed_body(RequestId, Body) -> {http, {RequestId, stream_end, _Headers}} -> Body; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end. receive_streamed_body(RequestId, Body, Pid) -> @@ -2959,7 +2995,7 @@ receive_streamed_body(RequestId, Body, Pid) -> {http, {RequestId, stream_end, _Headers}} -> Body; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end. %% Perform a synchronous stop @@ -3417,7 +3453,7 @@ handle_auth("Basic " ++ UserInfo, Challange, DefaultResponse) -> end. check_cookie([]) -> - test_server:fail(no_cookie_header); + tsf(no_cookie_header); check_cookie(["cookie:" ++ _Value | _]) -> ok; check_cookie([_Head | Tail]) -> @@ -3475,9 +3511,9 @@ p(F, A) -> io:format("~p ~w:" ++ F ++ "~n", [self(), ?MODULE | A]). tsp(F) -> - tsp(F, []). + inets_test_lib:tsp(F). tsp(F, A) -> - test_server:format("~p ~p:" ++ F ++ "~n", [self(), ?MODULE | A]). + inets_test_lib:tsp(F, A). tsf(Reason) -> test_server:fail(Reason). @@ -3532,3 +3568,6 @@ ensure_started(App) when is_atom(App) -> throw({error, {failed_starting, App, Error}}) end. + +skip(Reason) -> + {skip, Reason}. diff --git a/lib/inets/test/httpd_1_1.erl b/lib/inets/test/httpd_1_1.erl index 2a6110e3ea..07d94ea97a 100644 --- a/lib/inets/test/httpd_1_1.erl +++ b/lib/inets/test/httpd_1_1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,6 @@ %% -module(httpd_1_1). --author('[email protected]'). -include("test_server.hrl"). -include("test_server_line.hrl"). @@ -159,70 +158,79 @@ if_test(Type, Port, Host, Node, DocRoot)-> calendar:datetime_to_gregorian_seconds(FileInfo#file_info.mtime), Mod = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec-1)), - + CreatedSec-1)), + %% Test that we get the data when the file is modified ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ Host ++ - "\r\nIf-Modified-Since:" ++ - Mod ++ "\r\n\r\n", - [{statuscode, 200}]), - Mod1 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec+100)), - ok = httpd_test_lib:verify_request(Type,Host,Port,Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++"\r\nIf-Modified-Since:" - ++ Mod1 ++"\r\n\r\n", - [{statuscode, 304}]), + "GET / HTTP/1.1\r\nHost:" ++ Host ++ + "\r\nIf-Modified-Since:" ++ + Mod ++ "\r\n\r\n", + [{statuscode, 200}]), + Mod1 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( + CreatedSec+100)), + ok = httpd_test_lib:verify_request(Type,Host,Port,Node, + "GET / HTTP/1.1\r\nHost:" + ++ Host ++"\r\nIf-Modified-Since:" + ++ Mod1 ++"\r\n\r\n", + [{statuscode, 304}]), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "GET / HTTP/1.1\r\nHost:" ++ Host ++ + "\r\nIf-Modified-Since:" ++ + "AAA[...]AAAA" ++ "\r\n\r\n", + [{statuscode, 400}]), + + Mod2 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec+1)), + CreatedSec+1)), %% Control that the If-Unmodified-Header lmits the response ok = httpd_test_lib:verify_request(Type,Host,Port,Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:" ++ Mod2 - ++ "\r\n\r\n", - [{statuscode, 200}]), + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:" ++ Mod2 + ++ "\r\n\r\n", + [{statuscode, 200}]), Mod3 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec-1)), + CreatedSec-1)), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:"++ Mod3 - ++"\r\n\r\n", - [{statuscode, 412}]), - + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:"++ Mod3 + ++"\r\n\r\n", + [{statuscode, 412}]), + %% Control that we get the body when the etag match ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ Host - ++"\r\n"++ - "If-Match:"++ - httpd_util:create_etag(FileInfo)++ - "\r\n\r\n", - [{statuscode, 200}]), + "GET / HTTP/1.1\r\nHost:" ++ Host + ++"\r\n"++ + "If-Match:"++ + httpd_util:create_etag(FileInfo)++ + "\r\n\r\n", + [{statuscode, 200}]), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ - Host ++ "\r\n"++ - "If-Match:NotEtag\r\n\r\n", - [{statuscode, 412}]), + "GET / HTTP/1.1\r\nHost:" ++ + Host ++ "\r\n"++ + "If-Match:NotEtag\r\n\r\n", + [{statuscode, 412}]), %% Control the response when the if-none-match header is there ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++"\r\n"++ - "If-None-Match:NoTaag," ++ - httpd_util:create_etag(FileInfo) ++ - "\r\n\r\n", - [{statuscode, 304}]), - + "GET / HTTP/1.1\r\nHost:" + ++ Host ++"\r\n"++ + "If-None-Match:NoTaag," ++ + httpd_util:create_etag(FileInfo) ++ + "\r\n\r\n", + [{statuscode, 304}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ "\r\n"++ - "If-None-Match:NotEtag," - "NeihterEtag\r\n\r\n", - [{statuscode,200}]). + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ "\r\n"++ + "If-None-Match:NotEtag," + "NeihterEtag\r\n\r\n", + [{statuscode,200}]), + ok. http_trace(Type, Port, Host, Node)-> ok = httpd_test_lib:verify_request(Type, Host, Port, Node, diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index faeed3b5f9..ba31788ccc 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -686,6 +686,19 @@ end_per_testcase2(Case, Config) -> %%------------------------------------------------------------------------- +http_1_1_ip(doc) -> + ["HTTP/1.1"]; +http_1_1_ip(suite) -> + [ + ip_host, + ip_chunked, + ip_expect, + ip_range, + ip_if_test, + ip_http_trace, + ip_http1_1_head, + ip_mod_cgi_chunked_encoding_test + ]. %%------------------------------------------------------------------------- @@ -2276,24 +2289,24 @@ ticket_5913(doc) -> ["Tests that a header without last-modified is handled"]; ticket_5913(suite) -> []; ticket_5913(Config) -> - ok=httpd_test_lib:verify_request(ip_comm, ?config(host, Config), - ?IP_PORT, ?config(node, Config), + ok = httpd_test_lib:verify_request(ip_comm, ?config(host, Config), + ?IP_PORT, ?config(node, Config), "GET /cgi-bin/erl/httpd_example:get_bin " "HTTP/1.0\r\n\r\n", [{statuscode, 200}, - {version, "HTTP/1.0"}]), + {version, "HTTP/1.0"}]), ok. ticket_6003(doc) -> ["Tests that a URI with a bad hexadecimal code is handled"]; ticket_6003(suite) -> []; ticket_6003(Config) -> - ok=httpd_test_lib:verify_request(ip_comm, ?config(host, Config), - ?IP_PORT, ?config(node, Config), - "GET http://www.erlang.org/%skalle " - "HTTP/1.0\r\n\r\n", - [{statuscode, 400}, - {version, "HTTP/1.0"}]), + ok = httpd_test_lib:verify_request(ip_comm, ?config(host, Config), + ?IP_PORT, ?config(node, Config), + "GET http://www.erlang.org/%skalle " + "HTTP/1.0\r\n\r\n", + [{statuscode, 400}, + {version, "HTTP/1.0"}]), ok. ticket_7304(doc) -> diff --git a/lib/inets/test/httpd_basic_SUITE.erl b/lib/inets/test/httpd_basic_SUITE.erl index f23d0b4765..4cd38f2ec4 100644 --- a/lib/inets/test/httpd_basic_SUITE.erl +++ b/lib/inets/test/httpd_basic_SUITE.erl @@ -59,9 +59,28 @@ init_per_suite(Config) -> "~n Config: ~p", [Config]), ok = inets:start(), PrivDir = ?config(priv_dir, Config), - HttpdConf = [{port, 0}, {ipfamily, inet}, - {server_name, "httpd_test"}, {server_root, PrivDir}, - {document_root, PrivDir}, {bind_address, "localhost"}], + + Dummy = +"<HTML> +<HEAD> +<TITLE>/index.html</TITLE> +</HEAD> +<BODY> +DUMMY +</BODY> +</HTML>", + + DummyFile = filename:join([PrivDir,"dummy.html"]), + {ok, Fd} = file:open(DummyFile, [write]), + ok = file:write(Fd, Dummy), + ok = file:close(Fd), + HttpdConf = [{port, 0}, + {ipfamily, inet}, + {server_name, "httpd_test"}, + {server_root, PrivDir}, + {document_root, PrivDir}, + {bind_address, "localhost"}], + [{httpd_conf, HttpdConf} | Config]. %%-------------------------------------------------------------------- @@ -133,6 +152,10 @@ uri_too_long_414(Config) when is_list(Config) -> {version, "HTTP/0.9"}]), inets:stop(httpd, Pid). + +%%------------------------------------------------------------------------- +%%------------------------------------------------------------------------- + header_too_long_413(doc) -> ["Test that too long headers's get 413 HTTP code"]; header_too_long_413(suite) -> @@ -152,34 +175,92 @@ header_too_long_413(Config) when is_list(Config) -> {version, "HTTP/1.1"}]), inets:stop(httpd, Pid). + +%%------------------------------------------------------------------------- +%%------------------------------------------------------------------------- + escaped_url_in_error_body(doc) -> ["Test Url-encoding see OTP-8940"]; escaped_url_in_error_body(suite) -> []; escaped_url_in_error_body(Config) when is_list(Config) -> - tsp("escaped_url_in_error_body -> entry with" - "~n Config: ~p", [Config]), - HttpdConf = ?config(httpd_conf, Config), - {ok, Pid} = inets:start(httpd, [{port, 0} | HttpdConf]), - Info = httpd:info(Pid), - Port = proplists:get_value(port, Info), - _Address = proplists:get_value(bind_address, Info), - Path = "/<b>this_is_bold</b>", - URL = ?URL_START ++ integer_to_list(Port) ++ Path, - EscapedPath = http_uri:encode(Path), - {ok, {404, Body1}} = httpc:request(get, {URL, []}, - [{url_encode, true}, - {version, "HTTP/1.0"}], - [{full_result, false}]), - EscapedPath = find_URL_path(string:tokens(Body1, " ")), - {ok, {404, Body2}} = httpc:request(get, {URL, []}, - [{url_encode, false}, - {version, "HTTP/1.0"}], - [{full_result, false}]), + tsp("escaped_url_in_error_body -> entry"), + HttpdConf = ?config(httpd_conf, Config), + {ok, Pid} = inets:start(httpd, [{port, 0} | HttpdConf]), + Info = httpd:info(Pid), + Port = proplists:get_value(port, Info), + _Address = proplists:get_value(bind_address, Info), + + %% Request 1 + tsp("escaped_url_in_error_body -> request 1"), + URL1 = ?URL_START ++ integer_to_list(Port), + %% Make sure the server is ok, by making a request for a valid page + case httpc:request(get, {URL1 ++ "/dummy.html", []}, + [{url_encode, false}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {200, _}} -> + %% Don't care about the the body, just that we get a ok response + ok; + {ok, UnexpectedOK1} -> + tsf({unexpected_ok_1, UnexpectedOK1}) + end, + + %% Request 2 + tsp("escaped_url_in_error_body -> request 2"), + %% Make sure the server is ok, by making a request for a valid page + case httpc:request(get, {URL1 ++ "/dummy.html", []}, + [{url_encode, true}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {200, _}} -> + %% Don't care about the the body, just that we get a ok response + ok; + {ok, UnexpectedOK2} -> + tsf({unexpected_ok_2, UnexpectedOK2}) + end, + + %% Request 3 + tsp("escaped_url_in_error_body -> request 3"), + %% Ask for a non-existing page(1) + Path = "/<b>this_is_bold<b>", HTMLEncodedPath = http_util:html_encode(Path), - HTMLEncodedPath = find_URL_path(string:tokens(Body2, " ")), + URL2 = URL1 ++ Path, + case httpc:request(get, {URL2, []}, + [{url_encode, true}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {404, Body3}} -> + case find_URL_path(string:tokens(Body3, " ")) of + HTMLEncodedPath -> + ok; + BadPath3 -> + tsf({unexpected_path_3, HTMLEncodedPath, BadPath3}) + end; + {ok, UnexpectedOK3} -> + tsf({unexpected_ok_1, UnexpectedOK3}) + end, + + %% Request 4 + tsp("escaped_url_in_error_body -> request 4"), + %% Ask for a non-existing page(2) + case httpc:request(get, {URL2, []}, + [{url_encode, false}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {404, Body4}} -> + case find_URL_path(string:tokens(Body4, " ")) of + HTMLEncodedPath -> + ok; + BadPath4 -> + tsf({unexpected_path_2, HTMLEncodedPath, BadPath4}) + end; + {ok, UnexpectedOK4} -> + tsf({unexpected_ok_4, UnexpectedOK4}) + end, + tsp("escaped_url_in_error_body -> stop inets"), inets:stop(httpd, Pid), - tsp("escaped_url_in_error_body -> done"), + tsp("escaped_url_in_error_body -> done"), ok. find_URL_path([]) -> @@ -191,7 +272,14 @@ find_URL_path([_ | Rest]) -> tsp(F) -> - tsp(F, []). + inets_test_lib:tsp(F). tsp(F, A) -> - test_server:format("~p ~p:" ++ F ++ "~n", [self(), ?MODULE | A]). + inets_test_lib:tsp(F, A). + +tsf(Reason) -> + test_server:fail(Reason). + + +skip(Reason) -> + {skip, Reason}. diff --git a/lib/inets/test/httpd_mod.erl b/lib/inets/test/httpd_mod.erl index 1754cec7bc..5016cdb9e6 100644 --- a/lib/inets/test/httpd_mod.erl +++ b/lib/inets/test/httpd_mod.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 2005-2011. All Rights Reserved. -%% +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the @@ -88,13 +88,13 @@ actions(Type, Port, Host, Node) -> %%------------------------------------------------------------------------- security(ServerRoot, Type, Port, Host, Node) -> -%% io:format(user, "~w:security -> entry with" -%% "~n ServerRoot: ~p" -%% "~n Type: ~p" -%% "~n Port: ~p" -%% "~n Host: ~p" -%% "~n Node: ~p" -%% "~n", [?MODULE, ServerRoot, Type, Port, Host, Node]), + %% io:format(user, "~w:security -> entry with" + %% "~n ServerRoot: ~p" + %% "~n Type: ~p" + %% "~n Port: ~p" + %% "~n Host: ~p" + %% "~n Node: ~p" + %% "~n", [?MODULE, ServerRoot, Type, Port, Host, Node]), %% io:format(user, "~w:security -> register~n", [?MODULE]), global:register_name(mod_security_test, self()), % Receive events @@ -175,8 +175,8 @@ security(ServerRoot, Type, Port, Host, Node) -> [{"one",_, Port, OpenDir,_}] -> ok; Blocked -> - io:format(user, "~w:security -> Blocked: ~p" - "~n", [?MODULE, Blocked]), + %% io:format(user, "~w:security -> Blocked: ~p" + %% "~n", [?MODULE, Blocked]), exit({unexpected_blocked, Blocked}) end, @@ -917,11 +917,11 @@ list_users(Node, Root, _Host, Port, Dir) -> receive_security_event(Event, Node, Port) -> -%% io:format(user, "~w:receive_security_event -> entry with" -%% "~n Event: ~p" -%% "~n Node: ~p" -%% "~n Port: ~p" -%% "~n", [?MODULE, Event, Node, Port]), + %% io:format(user, "~w:receive_security_event -> entry with" + %% "~n Event: ~p" + %% "~n Node: ~p" + %% "~n Port: ~p" + %% "~n", [?MODULE, Event, Node, Port]), receive Event -> ok; diff --git a/lib/inets/test/httpd_test_lib.erl b/lib/inets/test/httpd_test_lib.erl index 581461fe03..1c7bb512cc 100644 --- a/lib/inets/test/httpd_test_lib.erl +++ b/lib/inets/test/httpd_test_lib.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 2001-2011. All Rights Reserved. -%% +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the @@ -140,6 +140,9 @@ request(#state{mfa = {Module, Function, Args}, HeadRequest = lists:sublist(RequestStr, 1, 4), receive {tcp, Socket, Data} -> + io:format("~p ~w[~w]request -> received (tcp) data" + "~n Data: ~p" + "~n", [self(), ?MODULE, ?LINE, Data]), print(tcp, Data, State), case Module:Function([Data | Args]) of {ok, Parsed} -> @@ -150,11 +153,19 @@ request(#state{mfa = {Module, Function, Args}, request(State#state{mfa = NewMFA}, TimeOut) end; {tcp_closed, Socket} when Function =:= whole_body -> + io:format("~p ~w[~w]request -> " + "received (tcp) closed when whole_body" + "~n", [self(), ?MODULE, ?LINE]), print(tcp, "closed", State), State#state{body = hd(Args)}; {tcp_closed, Socket} -> + io:format("~p ~w[~w]request -> received (tcp) closed" + "~n", [self(), ?MODULE, ?LINE]), test_server:fail(connection_closed); {tcp_error, Socket, Reason} -> + io:format("~p ~w[~w]request -> received (tcp) error" + "~n Reason: ~p" + "~n", [self(), ?MODULE, ?LINE, Reason]), test_server:fail({tcp_error, Reason}); {ssl, Socket, Data} -> print(ssl, Data, State), @@ -174,11 +185,21 @@ request(#state{mfa = {Module, Function, Args}, {ssl_error, Socket, Reason} -> test_server:fail({ssl_error, Reason}) after TimeOut -> + io:format("~p ~w[~w]request -> timeout" + "~n", [self(), ?MODULE, ?LINE]), test_server:fail(connection_timed_out) end. handle_http_msg({Version, StatusCode, ReasonPharse, Headers, Body}, State = #state{request = RequestStr}) -> + io:format("~p ~w[~w]handle_http_msg -> entry with" + "~n Version: ~p" + "~n StatusCode: ~p" + "~n ReasonPharse: ~p" + "~n Headers: ~p" + "~n Body: ~p" + "~n", [self(), ?MODULE, ?LINE, + Version, StatusCode, ReasonPharse, Headers, Body]), case is_expect(RequestStr) of true -> State#state{status_line = {Version, @@ -235,13 +256,14 @@ handle_http_body(Body, State = #state{headers = Headers, end. validate(RequestStr, #state{status_line = {Version, StatusCode, _}, - headers = Headers, - body = Body}, Options, N, P) -> + headers = Headers, + body = Body}, Options, N, P) -> %% tsp("validate -> entry with" %% "~n StatusCode: ~p" %% "~n Headers: ~p" %% "~n Body: ~p", [StatusCode, Headers, Body]), + check_version(Version, Options), case lists:keysearch(statuscode, 1, Options) of {value, _} -> @@ -255,6 +277,7 @@ validate(RequestStr, #state{status_line = {Version, StatusCode, _}, list_to_integer(Headers#http_response_h.'content-length'), Body). + %%-------------------------------------------------------------------- %% Internal functions %%------------------------------------------------------------------ @@ -263,21 +286,20 @@ check_version(Version, Options) -> {value, {version, Version}} -> ok; {value, {version, Ver}} -> - test_server:fail({wrong_version, [{got, Version}, - {expected, Ver}]}); + tsf({wrong_version, [{got, Version}, + {expected, Ver}]}); _ -> case Version of "HTTP/1.1" -> ok; _ -> - test_server:fail({wrong_version, [{got, Version}, - {expected, "HTTP/1.1"}]}) + tsf({wrong_version, [{got, Version}, + {expected, "HTTP/1.1"}]}) end end. check_status_code(StatusCode, [], Options) -> - test_server:fail({wrong_status_code, [{got, StatusCode}, - {expected, Options}]}); + tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}); check_status_code(StatusCode, Current = [_ | Rest], Options) -> case lists:keysearch(statuscode, 1, Current) of {value, {statuscode, StatusCode}} -> @@ -285,8 +307,7 @@ check_status_code(StatusCode, Current = [_ | Rest], Options) -> {value, {statuscode, _OtherStatus}} -> check_status_code(StatusCode, Rest, Options); false -> - test_server:fail({wrong_status_code, [{got, StatusCode}, - {expected, Options}]}) + tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}) end. do_validate(_, [], _, _) -> @@ -317,8 +338,7 @@ do_validate(Header, [{header, HeaderField, Value}|Rest],N,P) -> Header}) end, do_validate(Header, Rest, N, P); -do_validate(Header,[{no_last_modified,HeaderField}|Rest],N,P) -> -% io:format("Header: ~p~nHeaderField: ~p~n",[Header,HeaderField]), +do_validate(Header,[{no_last_modified, HeaderField}|Rest],N,P) -> case lists:keysearch(HeaderField,1,Header) of {value,_} -> test_server:fail({wrong_header_field_value, HeaderField, @@ -331,7 +351,6 @@ do_validate(Header, [_Unknown | Rest], N, P) -> do_validate(Header, Rest, N, P). is_expect(RequestStr) -> - case inets_regexp:match(RequestStr, "xpect:100-continue") of {match, _, _}-> true; @@ -340,15 +359,15 @@ is_expect(RequestStr) -> end. %% OTP-5775, content-length -check_body("GET /cgi-bin/erl/httpd_example:get_bin HTTP/1.0\r\n\r\n", 200, "text/html", Length, _Body) when Length /= 274-> - test_server:fail(content_length_error); +check_body("GET /cgi-bin/erl/httpd_example:get_bin HTTP/1.0\r\n\r\n", 200, "text/html", Length, _Body) when (Length =/= 274) -> + tsf(content_length_error); check_body("GET /cgi-bin/cgi_echo HTTP/1.0\r\n\r\n", 200, "text/plain", _, Body) -> case size(Body) of 100 -> ok; _ -> - test_server:fail(content_length_error) + tsf(content_length_error) end; check_body(RequestStr, 200, "text/html", _, Body) -> diff --git a/lib/kernel/doc/src/gen_sctp.xml b/lib/kernel/doc/src/gen_sctp.xml index 688cd0f78f..418bfae4b8 100644 --- a/lib/kernel/doc/src/gen_sctp.xml +++ b/lib/kernel/doc/src/gen_sctp.xml @@ -45,7 +45,15 @@ SUSE Linux Enterprise Server 10 (x86_64) kernel 2.6.16.27-0.6-smp, with lksctp-tools-1.0.6, briefly on Solaris 10, and later on SUSE Linux Enterprise Server 10 Service Pack 1 (x86_64) - kernel 2.6.16.54-0.2.3-smp with lksctp-tools-1.0.7.</p> + kernel 2.6.16.54-0.2.3-smp with lksctp-tools-1.0.7, + and later also on FreeBSD 8.2. + </p> + <p> + This module was written for one-to-many style sockets + (type <c>seqpacket</c>). With the addition of + <seealso marker="#peeloff/2">peeloff/2</seealso>, one-to-one style + sockets (type <c>stream</c>) were introduced. + </p> <p>Record definitions for the <c>gen_sctp</c> module can be found using:</p> <pre> -include_lib("kernel/include/inet_sctp.hrl"). </pre> <p>These record definitions use the "new" spelling 'adaptation', @@ -254,15 +262,19 @@ </desc> </func> <func> - <name name="listen" arity="2"/> + <name name="listen" arity="2" clause_i="1"/> + <name name="listen" arity="2" clause_i="2"/> <fsummary>Set up a socket to listen.</fsummary> <desc> <p>Sets up a socket to listen on the IP address and port number - it is bound to. <c><anno>IsServer</anno></c> must be <c>true</c> - or <c>false</c>. - In the contrast to TCP, in SCTP there is no listening queue length. - If <c><anno>IsServer</anno></c> is <c>true</c> the socket accepts new associations, i.e. - it will become an SCTP server socket.</p> + it is bound to.</p> + <p>For type <c>seqpacket</c> sockets (the default) + <c><anno>IsServer</anno></c> must be <c>true</c> or <c>false</c>. + In the contrast to TCP, in SCTP there is no listening queue length. + If <c><anno>IsServer</anno></c> is <c>true</c> the socket accepts new associations, i.e. + it will become an SCTP server socket.</p> + <p>For type <c>stream</c> sockets <anno>Backlog</anno> defines + the backlog queue length just like in TCP.</p> </desc> </func> <func> @@ -295,12 +307,40 @@ is used. In particular, the socket is opened in <seealso marker="#option-binary">binary</seealso> and <seealso marker="#option-active">passive</seealso> mode, + with <anno>SockType</anno> <c>seqpacket</c>, and with reasonably large <seealso marker="#option-sndbuf">kernel</seealso> and driver <seealso marker="#option-buffer">buffers.</seealso></p> </desc> </func> <func> + <name name="peeloff" arity="2"/> + <fsummary> + Peel off a type <c>stream</c> socket from a type <c>seqpacket</c> one + </fsummary> + <desc> + <p> + Branch off an existing association <anno>Assoc</anno> + in a socket <anno>Socket</anno> of type <c>seqpacket</c> + (one-to-may style) into + a new socket <anno>NewSocket</anno> of type <c>stream</c> + (one-to-one style). + </p> + <p> + The existing association argument <anno>Assoc</anno> + can be either a + <seealso marker="#record-sctp_assoc_change"> + #sctp_assoc_change{} + </seealso> + record as returned from e.g + <seealso marker="#recv-2">recv/*</seealso>, + <seealso marker="#connect-5">connect/*</seealso> or + from a listening socket in active mode. Or it can be just + the field <c>assoc_id</c> integer from such a record. + </p> + </desc> + </func> + <func> <name name="recv" arity="1"/> <name name="recv" arity="2"/> <fsummary>Receive a message from a socket</fsummary> diff --git a/lib/kernel/doc/src/heart.xml b/lib/kernel/doc/src/heart.xml index e2dbcbe63d..26d1e27822 100644 --- a/lib/kernel/doc/src/heart.xml +++ b/lib/kernel/doc/src/heart.xml @@ -42,7 +42,7 @@ system.</p> <p>An Erlang runtime system to be monitored by a heart program, should be started with the command line flag <c>-heart</c> (see - also <seealso marker="erts:erl">erl(1)</seealso>. The <c>heart</c> + also <seealso marker="erts:erl">erl(1)</seealso>). The <c>heart</c> process is then started automatically:</p> <pre> % <input>erl -heart ...</input></pre> diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index bf513b7815..0f71a4f0f2 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -4,7 +4,7 @@ <appref> <header> <copyright> - <year>1996</year><year>2009</year> + <year>1996</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -231,6 +231,15 @@ MaxT = TickTime + TickTime / 4</code> <p><em>Note:</em> Normally, a terminating node is detected immediately.</p> </item> + <tag><c>shutdown_timeout = integer() | infinity</c></tag> + <item> + <p>Specifies the time <c>application_controller</c> will wait + for an application to terminate during node shutdown. If the + timer expires, <c>application_controller</c> will brutally + kill <c>application_master</c> of the hanging + application. If this parameter is undefined, it defaults + to <c>infinity</c>.</p> + </item> <tag><c>sync_nodes_mandatory = [NodeName]</c></tag> <item> <p>Specifies which other nodes <em>must</em> be alive in order diff --git a/lib/kernel/doc/src/net_kernel.xml b/lib/kernel/doc/src/net_kernel.xml index 3b7a710664..e54a427ff0 100644 --- a/lib/kernel/doc/src/net_kernel.xml +++ b/lib/kernel/doc/src/net_kernel.xml @@ -210,6 +210,10 @@ <p><c>net_kernel</c> is currently changing <c>net_ticktime</c> to <c><anno>NetTicktime</anno></c> seconds.</p> </item> + <tag><c>ignored</c></tag> + <item> + <p>The local node is not alive.</p> + </item> </taglist> </desc> </func> diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 42f527f400..ebfe84463a 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1180,10 +1180,27 @@ terminate(Reason, S) -> _ -> ok end, + ShutdownTimeout = + case application:get_env(kernel, shutdown_timeout) of + undefined -> infinity; + {ok,T} -> T + end, foreach(fun({_AppName, Id}) when is_pid(Id) -> + Ref = erlang:monitor(process, Id), + unlink(Id), exit(Id, shutdown), receive + %% Proc died before link {'EXIT', Id, _} -> ok + after 0 -> + receive + {'DOWN', Ref, process, Id, _} -> ok + after ShutdownTimeout -> + exit(Id, kill), + receive + {'DOWN', Ref, process, Id, _} -> ok + end + end end; (_) -> ok end, diff --git a/lib/kernel/src/gen_sctp.erl b/lib/kernel/src/gen_sctp.erl index 6cebb7ab97..77ca26b845 100644 --- a/lib/kernel/src/gen_sctp.erl +++ b/lib/kernel/src/gen_sctp.erl @@ -27,7 +27,8 @@ -include("inet_sctp.hrl"). -export([open/0,open/1,open/2,close/1]). --export([listen/2,connect/4,connect/5,connect_init/4,connect_init/5]). +-export([listen/2,peeloff/2]). +-export([connect/4,connect/5,connect_init/4,connect_init/5]). -export([eof/2,abort/2]). -export([send/3,send/4,recv/1,recv/2]). -export([error_string/1]). @@ -109,9 +110,11 @@ open() -> | {ifaddr,IP} | inet:address_family() | {port,Port} + | {type,SockType} | option(), IP :: inet:ip_address() | any | loopback, Port :: inet:port_number(), + SockType :: seqpacket | stream, Socket :: sctp_socket(). open(Opts) when is_list(Opts) -> @@ -134,9 +137,11 @@ open(X) -> | {ifaddr,IP} | inet:address_family() | {port,Port} + | {type,SockType} | option(), IP :: inet:ip_address() | any | loopback, Port :: inet:port_number(), + SockType :: seqpacket | stream, Socket :: sctp_socket(). open(Port, Opts) when is_integer(Port), is_list(Opts) -> @@ -161,17 +166,38 @@ close(S) -> -spec listen(Socket, IsServer) -> ok | {error, Reason} when Socket :: sctp_socket(), IsServer :: boolean(), + Reason :: term(); + (Socket, Backlog) -> ok | {error, Reason} when + Socket :: sctp_socket(), + Backlog :: integer(), Reason :: term(). -listen(S, Flag) when is_port(S), is_boolean(Flag) -> +listen(S, Backlog) + when is_port(S), is_boolean(Backlog); + is_port(S), is_integer(Backlog) -> case inet_db:lookup_socket(S) of {ok,Mod} -> - Mod:listen(S, Flag); + Mod:listen(S, Backlog); Error -> Error end; listen(S, Flag) -> erlang:error(badarg, [S,Flag]). +-spec peeloff(Socket, Assoc) -> {ok, NewSocket} | {error, Reason} when + Socket :: sctp_socket(), + Assoc :: #sctp_assoc_change{} | assoc_id(), + NewSocket :: sctp_socket(), + Reason :: term(). + +peeloff(S, #sctp_assoc_change{assoc_id=AssocId}) when is_port(S) -> + peeloff(S, AssocId); +peeloff(S, AssocId) when is_port(S), is_integer(AssocId) -> + case inet_db:lookup_socket(S) of + {ok,Mod} -> + Mod:peeloff(S, AssocId); + Error -> Error + end. + -spec connect(Socket, Addr, Port, Opts) -> {ok, Assoc} | {error, inet:posix()} when Socket :: sctp_socket(), Addr :: inet:ip_address() | inet:hostname(), diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index 7d15f8bf83..fa97614eca 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -28,7 +28,7 @@ %% External exports -export([start/0, start_link/0, stop/0, sync/0, sync/1, - safe_whereis_name/1, whereis_name/1, register_name/2, + whereis_name/1, register_name/2, register_name/3, register_name_external/2, register_name_external/3, unregister_name_external/1,re_register_name/2, re_register_name/3, unregister_name/1, registered_names/0, send/2, node_disconnected/1, @@ -203,10 +203,6 @@ send(Name, Msg) -> whereis_name(Name) -> where(Name). --spec safe_whereis_name(term()) -> pid() | 'undefined'. -safe_whereis_name(Name) -> - gen_server:call(global_name_server, {whereis, Name}, infinity). - node_disconnected(Node) -> global_name_server ! {nodedown, Node}. @@ -510,8 +506,7 @@ init([]) -> %% delay can sometimes be quite substantial. Global guarantees that %% the name will eventually be removed, but there is no %% synchronization between nodes; the name can be removed from some -%% node(s) long before it is removed from other nodes. Using -%% safe_whereis_name is no cure. +%% node(s) long before it is removed from other nodes. %% %% - Global cannot handle problems with the distribution very well. %% Depending on the value of the kernel variable 'net_ticktime' long @@ -589,10 +584,6 @@ init([]) -> {'reply', term(), state()} | {'stop', 'normal', 'stopped', state()}. -handle_call({whereis, Name}, From, S) -> - do_whereis(Name, From), - {noreply, S}; - handle_call({registrar, Fun}, From, S) -> S#state.the_registrar ! {trans_all_known, Fun, From}, {noreply, S}; @@ -1235,7 +1226,15 @@ ins_name_ext(Name, Pid, Method, RegNode, FromPidOrNode, ExtraInfo, S0) -> where(Name) -> case ets:lookup(global_names, Name) of - [{_Name, Pid, _Method, _RPid, _Ref}] -> Pid; + [{_Name, Pid, _Method, _RPid, _Ref}] -> + if node(Pid) == node() -> + case is_process_alive(Pid) of + true -> Pid; + false -> undefined + end; + true -> + Pid + end; [] -> undefined end. diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index 48a6f3db65..b60c68e3a1 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -36,7 +36,7 @@ -export([i/0, i/1, i/2]). --export([getll/1, getfd/1, open/7, fdopen/5]). +-export([getll/1, getfd/1, open/8, fdopen/6]). -export([tcp_controlling_process/2, udp_controlling_process/2, tcp_close/1, udp_close/1]). @@ -115,7 +115,8 @@ 'mtu' | 'netmask' | 'flags' |'hwaddr'. -type address_family() :: 'inet' | 'inet6'. --type protocol_option() :: 'tcp' | 'udp' | 'sctp'. +-type socket_protocol() :: 'tcp' | 'udp' | 'sctp'. +-type socket_type() :: 'stream' | 'dgram' | 'seqpacket'. -type stat_option() :: 'recv_cnt' | 'recv_max' | 'recv_avg' | 'recv_oct' | 'recv_dvi' | 'send_cnt' | 'send_max' | 'send_avg' | 'send_oct' | 'send_pend'. @@ -748,6 +749,8 @@ sctp_opt([Opt|Opts], Mod, R, As) -> sctp_opt(Opts, Mod, R#sctp_opts{port=P}, As); Error -> Error end; + {type,Type} when Type =:= seqpacket; Type =:= stream -> + sctp_opt(Opts, Mod, R#sctp_opts{type=Type}, As); binary -> sctp_opt (Opts, Mod, R, As, mode, binary); list -> sctp_opt (Opts, Mod, R, As, mode, list); {sctp_module,_} -> sctp_opt (Opts, Mod, R, As); % Done with @@ -996,13 +999,14 @@ gethostbyaddr_tm_native(Addr, Timer, Opts) -> Addr :: ip_address(), Port :: port_number(), Opts :: [socket_setopt()], - Protocol :: protocol_option(), - Family :: 'inet' | 'inet6', + Protocol :: socket_protocol(), + Family :: address_family(), + Type :: socket_type(), Module :: atom()) -> {'ok', socket()} | {'error', posix()}. -open(Fd, Addr, Port, Opts, Protocol, Family, Module) when Fd < 0 -> - case prim_inet:open(Protocol, Family) of +open(Fd, Addr, Port, Opts, Protocol, Family, Type, Module) when Fd < 0 -> + case prim_inet:open(Protocol, Family, Type) of {ok,S} -> case prim_inet:setopts(S, Opts) of ok -> @@ -1029,18 +1033,19 @@ open(Fd, Addr, Port, Opts, Protocol, Family, Module) when Fd < 0 -> Error -> Error end; -open(Fd, _Addr, _Port, Opts, Protocol, Family, Module) -> - fdopen(Fd, Opts, Protocol, Family, Module). +open(Fd, _Addr, _Port, Opts, Protocol, Family, Type, Module) -> + fdopen(Fd, Opts, Protocol, Family, Type, Module). -spec fdopen(Fd :: non_neg_integer(), Opts :: [socket_setopt()], - Protocol :: protocol_option(), + Protocol :: socket_protocol(), Family :: address_family(), + Type :: socket_type(), Module :: atom()) -> {'ok', socket()} | {'error', posix()}. -fdopen(Fd, Opts, Protocol, Family, Module) -> - case prim_inet:fdopen(Protocol, Fd, Family) of +fdopen(Fd, Opts, Protocol, Family, Type, Module) -> + case prim_inet:fdopen(Protocol, Family, Type, Fd) of {ok, S} -> case prim_inet:setopts(S, Opts) of ok -> @@ -1056,18 +1061,24 @@ fdopen(Fd, Opts, Protocol, Family, Module) -> %% socket stat %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -i() -> i(tcp), i(udp). +i() -> i(tcp), i(udp), i(sctp). i(Proto) -> i(Proto, [port, module, recv, sent, owner, - local_address, foreign_address, state]). + local_address, foreign_address, state, type]). i(tcp, Fs) -> ii(tcp_sockets(), Fs, tcp); i(udp, Fs) -> - ii(udp_sockets(), Fs, udp). + ii(udp_sockets(), Fs, udp); +i(sctp, Fs) -> + ii(sctp_sockets(), Fs, sctp). ii(Ss, Fs, Proto) -> - LLs = [h_line(Fs) | info_lines(Ss, Fs, Proto)], + LLs = + case info_lines(Ss, Fs, Proto) of + [] -> []; + InfoLines -> [h_line(Fs) | InfoLines] + end, Maxs = foldl( fun(Line,Max0) -> smax(Max0,Line) end, duplicate(length(Fs),0),LLs), @@ -1135,6 +1146,7 @@ info(S, F, Proto) -> case prim_inet:gettype(S) of {ok,{_,stream}} -> "STREAM"; {ok,{_,dgram}} -> "DGRAM"; + {ok,{_,seqpacket}} -> "SEQPACKET"; _ -> " " end; fd -> @@ -1186,6 +1198,7 @@ fmt_port(N, Proto) -> %% Return a list of all tcp sockets tcp_sockets() -> port_list("tcp_inet"). udp_sockets() -> port_list("udp_inet"). +sctp_sockets() -> port_list("sctp_inet"). %% Return all ports having the name 'Name' port_list(Name) -> diff --git a/lib/kernel/src/inet6_sctp.erl b/lib/kernel/src/inet6_sctp.erl index 5bf3fca647..c47483bbdd 100644 --- a/lib/kernel/src/inet6_sctp.erl +++ b/lib/kernel/src/inet6_sctp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -32,7 +32,8 @@ -define(FAMILY, inet6). -export([getserv/1,getaddr/1,getaddr/2,translate_ip/1]). --export([open/1,close/1,listen/2,connect/5,sendmsg/3,send/4,recv/2]). +-export([open/1,close/1,listen/2,peeloff/2,connect/5]). +-export([sendmsg/3,send/4,recv/2]). @@ -54,8 +55,8 @@ translate_ip(IP) -> open(Opts) -> case inet:sctp_options(Opts, ?MODULE) of - {ok,#sctp_opts{fd=Fd,ifaddr=Addr,port=Port,opts=SOs}} -> - inet:open(Fd, Addr, Port, SOs, sctp, ?FAMILY, ?MODULE); + {ok,#sctp_opts{fd=Fd,ifaddr=Addr,port=Port,type=Type,opts=SOs}} -> + inet:open(Fd, Addr, Port, SOs, sctp, ?FAMILY, Type, ?MODULE); Error -> Error end. @@ -65,6 +66,14 @@ close(S) -> listen(S, Flag) -> prim_inet:listen(S, Flag). +peeloff(S, AssocId) -> + case prim_inet:peeloff(S, AssocId) of + {ok, NewS}=Result -> + inet_db:register_socket(NewS, ?MODULE), + Result; + Error -> Error + end. + connect(S, Addr, Port, Opts, Timer) -> inet_sctp:connect(S, Addr, Port, Opts, Timer). diff --git a/lib/kernel/src/inet6_tcp.erl b/lib/kernel/src/inet6_tcp.erl index cc45f6c7f6..c714b2bee0 100644 --- a/lib/kernel/src/inet6_tcp.erl +++ b/lib/kernel/src/inet6_tcp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -93,7 +93,7 @@ do_connect(Addr = {A,B,C,D,E,F,G,H}, Port, Opts, Time) when port=BPort, opts=SockOpts}} when ?ip6(Ab,Bb,Cb,Db,Eb,Fb,Gb,Hb), ?port(BPort) -> - case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet6,?MODULE) of + case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet6,stream,?MODULE) of {ok, S} -> case prim_inet:connect(S, Addr, Port, Time) of ok -> {ok,S}; @@ -115,7 +115,7 @@ listen(Port, Opts) -> port=BPort, opts=SockOpts}=R} when ?ip6(A,B,C,D,E,F,G,H), ?port(BPort) -> - case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet6,?MODULE) of + case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet6,stream,?MODULE) of {ok, S} -> case prim_inet:listen(S, R#listen_opts.backlog) of ok -> {ok, S}; @@ -149,5 +149,5 @@ accept(L,Timeout) -> %% Create a port/socket from a file descriptor %% fdopen(Fd, Opts) -> - inet:fdopen(Fd, Opts, tcp, inet6, ?MODULE). + inet:fdopen(Fd, Opts, tcp, inet6, stream, ?MODULE). diff --git a/lib/kernel/src/inet6_udp.erl b/lib/kernel/src/inet6_udp.erl index e81d417151..ca43c94211 100644 --- a/lib/kernel/src/inet6_udp.erl +++ b/lib/kernel/src/inet6_udp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -45,7 +45,7 @@ open(Port, Opts) -> port=BPort, opts=SockOpts}} when ?ip6(A,B,C,D,E,F,G,H), ?port(BPort) -> - inet:open(Fd,BAddr,BPort,SockOpts,udp,inet6,?MODULE); + inet:open(Fd,BAddr,BPort,SockOpts,udp,inet6,dgram,?MODULE); {ok, _} -> exit(badarg) end. @@ -84,4 +84,4 @@ controlling_process(Socket, NewOwner) -> %% Create a port/socket from a file descriptor %% fdopen(Fd, Opts) -> - inet:fdopen(Fd, Opts, udp, inet6, ?MODULE). + inet:fdopen(Fd, Opts, udp, inet6, dgram, ?MODULE). diff --git a/lib/kernel/src/inet_int.hrl b/lib/kernel/src/inet_int.hrl index 6f1688c6a2..f8984b13fe 100644 --- a/lib/kernel/src/inet_int.hrl +++ b/lib/kernel/src/inet_int.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -define(INET_AF_ANY, 3). % Fake for ANY in any address family -define(INET_AF_LOOPBACK, 4). % Fake for LOOPBACK in any address family -%% type codes (gettype, INET_REQ_GETTYPE) +%% type codes to open and gettype - INET_REQ_GETTYPE -define(INET_TYPE_STREAM, 1). -define(INET_TYPE_DGRAM, 2). -define(INET_TYPE_SEQPACKET, 3). @@ -83,16 +83,19 @@ -define(INET_REQ_IFSET, 23). -define(INET_REQ_SUBSCRIBE, 24). -define(INET_REQ_GETIFADDRS, 25). +-define(INET_REQ_ACCEPT, 26). +-define(INET_REQ_LISTEN, 27). %% TCP requests --define(TCP_REQ_ACCEPT, 40). --define(TCP_REQ_LISTEN, 41). +%%-define(TCP_REQ_ACCEPT, 40). MOVED +%%-define(TCP_REQ_LISTEN, 41). MERGED -define(TCP_REQ_RECV, 42). -define(TCP_REQ_UNRECV, 43). -define(TCP_REQ_SHUTDOWN, 44). %% UDP and SCTP requests -define(PACKET_REQ_RECV, 60). --define(SCTP_REQ_LISTEN, 61). +%%-define(SCTP_REQ_LISTEN, 61). MERGED -define(SCTP_REQ_BINDX, 62). %% Multi-home SCTP bind +-define(SCTP_REQ_PEELOFF, 63). %% subscribe codes, INET_REQ_SUBSCRIBE -define(INET_SUBS_EMPTY_OUT_Q, 1). @@ -100,7 +103,7 @@ %% reply codes for *_REQ_* -define(INET_REP_ERROR, 0). -define(INET_REP_OK, 1). --define(INET_REP_SCTP, 2). +-define(INET_REP, 2). %% INET, TCP and UDP options: -define(INET_OPT_REUSEADDR, 0). @@ -399,6 +402,7 @@ ifaddr, port = 0, fd = -1, + type = seqpacket, opts = [{mode, binary}, {buffer, ?SCTP_DEF_BUFSZ}, {sndbuf, ?SCTP_DEF_BUFSZ}, diff --git a/lib/kernel/src/inet_sctp.erl b/lib/kernel/src/inet_sctp.erl index de74b573bd..2d799d79fa 100644 --- a/lib/kernel/src/inet_sctp.erl +++ b/lib/kernel/src/inet_sctp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -31,7 +31,8 @@ -define(FAMILY, inet). -export([getserv/1,getaddr/1,getaddr/2,translate_ip/1]). --export([open/1,close/1,listen/2,connect/5,sendmsg/3,send/4,recv/2]). +-export([open/1,close/1,listen/2,peeloff/2,connect/5]). +-export([sendmsg/3,send/4,recv/2]). @@ -53,8 +54,8 @@ translate_ip(IP) -> open(Opts) -> case inet:sctp_options(Opts, ?MODULE) of - {ok,#sctp_opts{fd=Fd,ifaddr=Addr,port=Port,opts=SOs}} -> - inet:open(Fd, Addr, Port, SOs, sctp, ?FAMILY, ?MODULE); + {ok,#sctp_opts{fd=Fd,ifaddr=Addr,port=Port,type=Type,opts=SOs}} -> + inet:open(Fd, Addr, Port, SOs, sctp, ?FAMILY, Type, ?MODULE); Error -> Error end. @@ -64,6 +65,14 @@ close(S) -> listen(S, Flag) -> prim_inet:listen(S, Flag). +peeloff(S, AssocId) -> + case prim_inet:peeloff(S, AssocId) of + {ok, NewS}=Result -> + inet_db:register_socket(NewS, ?MODULE), + Result; + Error -> Error + end. + %% A non-blocking connect is implemented when the initial call is to %% gen_sctp:connect_init which passes the value nowait as the Timer connect(S, Addr, Port, Opts, Timer) -> @@ -102,7 +111,7 @@ connect(S, Addr, Port, Opts, Timer) -> connect_get_assoc(S, Addr, Port, false, Timer) -> case recv(S, inet:timeout(Timer)) of - {ok, {Addr, Port, [], #sctp_assoc_change{state=St}=Ev}} -> + {ok, {Addr, Port, _, #sctp_assoc_change{state=St}=Ev}} -> if St =:= comm_up -> %% Yes, successfully connected, return the whole %% sctp_assoc_change event (containing, in particular, @@ -123,7 +132,7 @@ connect_get_assoc(S, Addr, Port, false, Timer) -> connect_get_assoc(S, Addr, Port, Active, Timer) -> Timeout = inet:timeout(Timer), receive - {sctp,S,Addr,Port,{[],#sctp_assoc_change{state=St}=Ev}} -> + {sctp,S,Addr,Port,{_,#sctp_assoc_change{state=St}=Ev}} -> case Active of once -> prim_inet:setopt(S, active, once); diff --git a/lib/kernel/src/inet_tcp.erl b/lib/kernel/src/inet_tcp.erl index 6dadccd6a9..4c2db16ce3 100644 --- a/lib/kernel/src/inet_tcp.erl +++ b/lib/kernel/src/inet_tcp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -95,7 +95,7 @@ do_connect({A,B,C,D}, Port, Opts, Time) when ?ip(A,B,C,D), ?port(Port) -> port=BPort, opts=SockOpts}} when ?ip(Ab,Bb,Cb,Db), ?port(BPort) -> - case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet,?MODULE) of + case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet,stream,?MODULE) of {ok, S} -> case prim_inet:connect(S, {A,B,C,D}, Port, Time) of ok -> {ok,S}; @@ -117,7 +117,7 @@ listen(Port, Opts) -> port=BPort, opts=SockOpts}=R} when ?ip(A,B,C,D), ?port(BPort) -> - case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet,?MODULE) of + case inet:open(Fd,BAddr,BPort,SockOpts,tcp,inet,stream,?MODULE) of {ok, S} -> case prim_inet:listen(S, R#listen_opts.backlog) of ok -> {ok, S}; @@ -150,4 +150,4 @@ accept(L,Timeout) -> %% Create a port/socket from a file descriptor %% fdopen(Fd, Opts) -> - inet:fdopen(Fd, Opts, tcp, inet, ?MODULE). + inet:fdopen(Fd, Opts, tcp, inet, stream, ?MODULE). diff --git a/lib/kernel/src/inet_udp.erl b/lib/kernel/src/inet_udp.erl index 60bd96f332..80d930fe10 100644 --- a/lib/kernel/src/inet_udp.erl +++ b/lib/kernel/src/inet_udp.erl @@ -52,7 +52,7 @@ open(Port, Opts) -> ifaddr=BAddr={A,B,C,D}, port=BPort, opts=SockOpts}} when ?ip(A,B,C,D), ?port(BPort) -> - inet:open(Fd,BAddr,BPort,SockOpts,udp,inet,?MODULE); + inet:open(Fd,BAddr,BPort,SockOpts,udp,inet,dgram,?MODULE); {ok, _} -> exit(badarg) end. @@ -93,7 +93,7 @@ controlling_process(Socket, NewOwner) -> fdopen(Fd, Opts) -> inet:fdopen(Fd, optuniquify([{recbuf, ?RECBUF} | Opts]), - udp, inet, ?MODULE). + udp, inet, dgram, ?MODULE). %% Remove all duplicate options from an option list. diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 2c5b8ccb66..f469a0af98 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -33,7 +33,7 @@ -export([config_change/1, distr_changed_tc1/1, distr_changed_tc2/1, - shutdown_func/1, do_shutdown/1]). + shutdown_func/1, do_shutdown/1, shutdown_timeout/1]). -define(TESTCASE, testcase_name). -define(testcase, ?config(?TESTCASE, Config)). @@ -50,7 +50,7 @@ all() -> load_use_cache, {group, reported_bugs}, start_phases, script_start, nodedown_start, permit_false_start_local, permit_false_start_dist, get_key, - {group, distr_changed}, config_change, shutdown_func]. + {group, distr_changed}, config_change, shutdown_func, shutdown_timeout]. groups() -> [{reported_bugs, [], @@ -1915,6 +1915,32 @@ do_shutdown(Reason) -> +%%%----------------------------------------------------------------- +%%% Tests the 'shutdown_timeout' kernel config parameter +%%%----------------------------------------------------------------- +shutdown_timeout(Config) when is_list(Config) -> + DataDir = ?config(data_dir,Config), + {ok,Cp1} = start_node(?MODULE_STRING++"_shutdown_timeout"), + wait_for_ready_net(), + ok = rpc:call(Cp1, application, set_env, [kernel, shutdown_timeout, 1000]), + rpc:call(Cp1, code, add_path, [filename:join([DataDir,deadlock])]), + ok = rpc:call(Cp1, application, start, [sasl]), + ok = rpc:call(Cp1, application, start, [deadlock]), + rpc:call(Cp1, deadlock, restart_and_fail, []), + + ok = net_kernel:monitor_nodes(true), + _ = rpc:call(Cp1, init, stop, []), + receive + {nodedown,Cp1} -> + ok + after 10000 -> + ct:fail("timeout 10 sec: node termination hangs") + end, + ok. + + + + %%----------------------------------------------------------------- %% Utility functions %%----------------------------------------------------------------- diff --git a/lib/kernel/test/application_SUITE_data/Makefile.src b/lib/kernel/test/application_SUITE_data/Makefile.src index a237f6badb..abc3c82907 100644 --- a/lib/kernel/test/application_SUITE_data/Makefile.src +++ b/lib/kernel/test/application_SUITE_data/Makefile.src @@ -2,7 +2,8 @@ EFLAGS=+debug_info all: app_start_error.@EMULATOR@ trans_abnormal_sup.@EMULATOR@ \ trans_normal_sup.@EMULATOR@ transient.@EMULATOR@ \ - group_leader_sup.@EMULATOR@ group_leader.@EMULATOR@ + group_leader_sup.@EMULATOR@ group_leader.@EMULATOR@ \ + deadlock/deadlock.@EMULATOR@ app_start_error.@EMULATOR@: app_start_error.erl erlc $(EFLAGS) app_start_error.erl @@ -22,3 +23,5 @@ group_leader.@EMULATOR@: group_leader.erl group_leader_sup.@EMULATOR@: group_leader_sup.erl erlc $(EFLAGS) group_leader_sup.erl +deadlock/deadlock.@EMULATOR@: deadlock/deadlock.erl + erlc $(EFLAGS) -o deadlock deadlock/deadlock.erl
\ No newline at end of file diff --git a/lib/kernel/test/application_SUITE_data/deadlock/deadlock.app b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.app new file mode 100644 index 0000000000..0c1001bed6 --- /dev/null +++ b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.app @@ -0,0 +1,8 @@ +{application, deadlock, [ + {vsn, "1"}, + {registered, []}, + {applications, [kernel, stdlib, sasl]}, + {modules, [deadlock]}, + {mod, {deadlock, []}}, + {env, [{fail_start, false}]} +]}. diff --git a/lib/kernel/test/application_SUITE_data/deadlock/deadlock.erl b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.erl new file mode 100644 index 0000000000..5f68bf9078 --- /dev/null +++ b/lib/kernel/test/application_SUITE_data/deadlock/deadlock.erl @@ -0,0 +1,69 @@ +-module(deadlock). +-behaviour(application). +-compile(export_all). +-define(SUP,deadlock_sup). +-define(CHILD,deadlock_child). + + +%%%----------------------------------------------------------------- +%%% application callbacks +start(_StartType, _StartArgs) -> + supervisor:start_link({local, ?SUP}, ?MODULE, [sup]). + +stop(_State) -> + ok. + + + +%%%----------------------------------------------------------------- +%%% supervisor callbacks +init([sup]) -> + {ok, {{one_for_one, 5, 10}, [ + { + sasl_syslog_dm, {?MODULE, start_link, []}, + permanent, brutal_kill, worker, + [deadlock] + } + ]}}; + + +%%%----------------------------------------------------------------- +%%% gen_server callbacks +init([child]) -> + case application:get_env(deadlock, fail_start) of + {ok, false} -> + %% we must not fail on the first init, otherwise supervisor + %% terminates immediately + {ok, []}; + {ok, true} -> + timer:sleep(infinity), % init hangs!!!! + {ok, []} + end. + +handle_call(_Req, _From, State) -> + {reply, ok, State}. + +handle_cast(restart, State) -> + {stop, error, State}. + +handle_info(_Msg, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%%----------------------------------------------------------------- +%%% Start child +start_link() -> + gen_server:start_link({local, ?CHILD}, ?MODULE, [child], []). + + +%%%----------------------------------------------------------------- +%%% Provoke hanging +restart_and_fail() -> + application:set_env(deadlock, fail_start, true), % next init will hang + gen_server:cast(?CHILD, restart). diff --git a/lib/kernel/test/erl_boot_server_SUITE.erl b/lib/kernel/test/erl_boot_server_SUITE.erl index cea3715ce4..bb64c01058 100644 --- a/lib/kernel/test/erl_boot_server_SUITE.erl +++ b/lib/kernel/test/erl_boot_server_SUITE.erl @@ -346,7 +346,7 @@ good_hosts(_Config) -> [GoodHost1, GoodHost2, GoodHost3]. open_udp() -> - ?line {ok, S} = prim_inet:open(udp, inet), + ?line {ok, S} = prim_inet:open(udp, inet, dgram), ?line ok = prim_inet:setopts(S, [{mode,list},{active,true}, {deliver,term},{broadcast,true}]), ?line {ok,_} = prim_inet:bind(S, {0,0,0,0}, 0), diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index 1b534a5fc4..300152ddce 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -30,33 +30,29 @@ -export( [basic/1, api_open_close/1,api_listen/1,api_connect_init/1,api_opts/1, - xfer_min/1,xfer_active/1,def_sndrcvinfo/1,implicit_inet6/1]). + xfer_min/1,xfer_active/1,def_sndrcvinfo/1,implicit_inet6/1, + basic_stream/1, xfer_stream_min/1, peeloff/1, buffers/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [basic, api_open_close, api_listen, api_connect_init, - api_opts, xfer_min, xfer_active, def_sndrcvinfo, - implicit_inet6]. + api_opts, xfer_min, xfer_active, def_sndrcvinfo, implicit_inet6, + basic_stream, xfer_stream_min, peeloff, buffers]. groups() -> []. -init_per_suite(Config) -> - try gen_sctp:open() of +init_per_suite(_Config) -> + case gen_sctp:open() of {ok,Socket} -> gen_sctp:close(Socket), []; - _ -> - [] - catch - error:badarg -> - {skip,"SCTP not supported on this machine"}; - _:_ -> - Config + {error,eprotonosupport} -> + {skip,"SCTP not supported on this machine"} end. -end_per_suite(_Conifig) -> +end_per_suite(_Config) -> ok. init_per_group(_GroupName, Config) -> @@ -96,7 +92,7 @@ xfer_min(Config) when is_list(Config) -> ?line Stream = 0, ?line Data = <<"The quick brown fox jumps over a lazy dog 0123456789">>, ?line Loopback = {127,0,0,1}, - ?line {ok,Sb} = gen_sctp:open(), + ?line {ok,Sb} = gen_sctp:open([{type,seqpacket}]), ?line {ok,Pb} = inet:port(Sb), ?line ok = gen_sctp:listen(Sb, true), @@ -108,29 +104,44 @@ xfer_min(Config) when is_list(Config) -> inbound_streams=SaInboundStreams, assoc_id=SaAssocId}=SaAssocChange} = gen_sctp:connect(Sa, Loopback, Pb, []), - ?line {ok,{Loopback, - Pa,[], + ?line {SbAssocId,SaOutboundStreams,SaInboundStreams} = + case recv_event(log_ok(gen_sctp:recv(Sb, infinity))) of + {Loopback,Pa, #sctp_assoc_change{state=comm_up, error=0, outbound_streams=SbOutboundStreams, inbound_streams=SbInboundStreams, - assoc_id=SbAssocId}}} = - gen_sctp:recv(Sb, infinity), - ?line SaOutboundStreams = SbInboundStreams, - ?line SbOutboundStreams = SaInboundStreams, + assoc_id=AssocId}} -> + {AssocId,SbInboundStreams,SbOutboundStreams}; + {Loopback,Pa, + #sctp_paddr_change{state=addr_confirmed, + addr={Loopback,Pa}, + error=0, + assoc_id=AssocId}} -> + {Loopback,Pa, + #sctp_assoc_change{state=comm_up, + error=0, + outbound_streams=SbOutboundStreams, + inbound_streams=SbInboundStreams, + assoc_id=AssocId}} = + ?line recv_event(log_ok(gen_sctp:recv(Sb, infinity))), + {AssocId,SbInboundStreams,SbOutboundStreams} + end, + ?line ok = gen_sctp:send(Sa, SaAssocId, 0, Data), - ?line case gen_sctp:recv(Sb, infinity) of - {ok,{Loopback, - Pa, - [#sctp_sndrcvinfo{stream=Stream, - assoc_id=SbAssocId}], - Data}} -> ok; - {ok,{Loopback, - Pa,[], + ?line case log_ok(gen_sctp:recv(Sb, infinity)) of + {Loopback, + Pa, + [#sctp_sndrcvinfo{stream=Stream, + assoc_id=SbAssocId}], + Data} -> ok; + Event1 -> + {Loopback,Pa, #sctp_paddr_change{addr = {Loopback,_}, state = addr_available, error = 0, - assoc_id = SbAssocId}}} -> + assoc_id = SbAssocId}} = + recv_event(Event1), {ok,{Loopback, Pa, [#sctp_sndrcvinfo{stream=Stream, @@ -138,30 +149,40 @@ xfer_min(Config) when is_list(Config) -> Data}} = gen_sctp:recv(Sb, infinity) end, ?line ok = gen_sctp:send(Sb, SbAssocId, 0, Data), - ?line {ok,{Loopback, - Pb, + ?line case log_ok(gen_sctp:recv(Sa, infinity)) of + {Loopback,Pb, [#sctp_sndrcvinfo{stream=Stream, assoc_id=SaAssocId}], - Data}} = - gen_sctp:recv(Sa, infinity), + Data} -> + ok; + Event2 -> + {Loopback,Pb, + #sctp_paddr_change{addr={_,Pb}, + state=addr_confirmed, + error=0, + assoc_id=SaAssocId}} = + ?line recv_event(Event2), + ?line {Loopback, + Pb, + [#sctp_sndrcvinfo{stream=Stream, + assoc_id=SaAssocId}], + Data} = + log_ok(gen_sctp:recv(Sa, infinity)) + end, %% ?line ok = gen_sctp:eof(Sa, SaAssocChange), - ?line {ok,{Loopback, - Pa,[], - #sctp_shutdown_event{assoc_id=SbAssocId}}} = - gen_sctp:recv(Sb, infinity), - ?line {ok,{Loopback, - Pb,[], - #sctp_assoc_change{state=shutdown_comp, - error=0, - assoc_id=SaAssocId}}} = - gen_sctp:recv(Sa, infinity), - ?line {ok,{Loopback, - Pa,[], - #sctp_assoc_change{state=shutdown_comp, - error=0, - assoc_id=SbAssocId}}} = - gen_sctp:recv(Sb, infinity), + ?line {Loopback,Pa,#sctp_shutdown_event{assoc_id=SbAssocId}} = + recv_event(log_ok(gen_sctp:recv(Sb, infinity))), + ?line {Loopback,Pb, + #sctp_assoc_change{state=shutdown_comp, + error=0, + assoc_id=SaAssocId}} = + recv_event(log_ok(gen_sctp:recv(Sa, infinity))), + ?line {Loopback,Pa, + #sctp_assoc_change{state=shutdown_comp, + error=0, + assoc_id=SbAssocId}} = + recv_event(log_ok(gen_sctp:recv(Sb, infinity))), ?line ok = gen_sctp:close(Sa), ?line ok = gen_sctp:close(Sb), @@ -186,32 +207,52 @@ xfer_active(Config) when is_list(Config) -> ?line {ok,Sa} = gen_sctp:open([{active,true}]), ?line {ok,Pa} = inet:port(Sa), - ?line {ok,#sctp_assoc_change{state=comm_up, - error=0, - outbound_streams=SaOutboundStreams, - inbound_streams=SaInboundStreams, - assoc_id=SaAssocId}=SaAssocChange} = - gen_sctp:connect(Sa, Loopback, Pb, []), + ?line ok = gen_sctp:connect_init(Sa, Loopback, Pb, []), + ?line #sctp_assoc_change{state=comm_up, + error=0, + outbound_streams=SaOutboundStreams, + inbound_streams=SaInboundStreams, + assoc_id=SaAssocId} = SaAssocChange = + recv_assoc_change(Sa, Loopback, Pb, Timeout), ?line io:format("Sa=~p, Pa=~p, Sb=~p, Pb=~p, SaAssocId=~p, " "SaOutboundStreams=~p, SaInboundStreams=~p~n", [Sa,Pa,Sb,Pb,SaAssocId, SaOutboundStreams,SaInboundStreams]), - ?line SbAssocId = - receive - {sctp,Sb,Loopback,Pa, - {[], - #sctp_assoc_change{state=comm_up, - error=0, - outbound_streams=SbOutboundStreams, - inbound_streams=SbInboundStreams, - assoc_id=SBAI}}} -> - ?line SaOutboundStreams = SbInboundStreams, - ?line SaInboundStreams = SbOutboundStreams, - SBAI - after Timeout -> - ?line test_server:fail({unexpected,flush()}) - end, + ?line #sctp_assoc_change{state=comm_up, + error=0, + outbound_streams=SbOutboundStreams, + inbound_streams=SbInboundStreams, + assoc_id=SbAssocId} = + recv_assoc_change(Sb, Loopback, Pa, Timeout), + ?line SbOutboundStreams = SaInboundStreams, + ?line SbInboundStreams = SaOutboundStreams, ?line io:format("SbAssocId=~p~n", [SbAssocId]), + + ?line case recv_paddr_change(Sa, Loopback, Pb, 314) of + #sctp_paddr_change{state=addr_confirmed, + addr={_,Pb}, + error=0, + assoc_id=SaAssocId} -> ok; + #sctp_paddr_change{state=addr_available, + addr={_,Pb}, + error=0, + assoc_id=SaAssocId} -> ok; + timeout -> ok + end, + ?line case recv_paddr_change(Sb, Loopback, Pa, 314) of + #sctp_paddr_change{state=addr_confirmed, + addr={Loopback,Pa}, + error=0, + assoc_id=SbAssocId} -> ok; + #sctp_paddr_change{state=addr_available, + addr={Loopback,P}, + error=0, + assoc_id=SbAssocId} -> + ?line match_unless_solaris(Pa, P); + timeout -> ok + end, + ?line [] = flush(), + ?line ok = do_from_other_process( fun () -> gen_sctp:send(Sa, SaAssocId, 0, Data) end), @@ -219,21 +260,9 @@ xfer_active(Config) when is_list(Config) -> {sctp,Sb,Loopback,Pa, {[#sctp_sndrcvinfo{stream=Stream, assoc_id=SbAssocId}], - Data}} -> ok; - {sctp,Sb,Loopback,Pa, - {[], - #sctp_paddr_change{addr = {Loopback,_}, - state = addr_available, - error = 0, - assoc_id = SbAssocId}}} -> - ?line receive - {sctp,Sb,Loopback,Pa, - {[#sctp_sndrcvinfo{stream=Stream, - assoc_id=SbAssocId}], - Data}} -> ok - end + Data}} -> ok after Timeout -> - ?line test_server:fail({unexpected,flush()}) + ?line test_server:fail({timeout,flush()}) end, ?line ok = gen_sctp:send(Sb, SbAssocId, 0, Data), ?line receive @@ -242,31 +271,28 @@ xfer_active(Config) when is_list(Config) -> assoc_id=SaAssocId}], Data}} -> ok after Timeout -> - ?line test_server:fail({unexpected,flush()}) + ?line test_server:fail({timeout,flush()}) end, %% ?line ok = gen_sctp:abort(Sa, SaAssocChange), - ?line receive - {sctp,Sb,Loopback,Pa, - {[], - #sctp_assoc_change{state=comm_lost, - assoc_id=SbAssocId}}} -> ok - after Timeout -> - ?line test_server:fail({unexpected,flush()}) + ?line case recv_assoc_change(Sb, Loopback, Pa, Timeout) of + #sctp_assoc_change{state=comm_lost, + assoc_id=SbAssocId} -> ok; + timeout -> + ?line test_server:fail({timeout,flush()}) end, ?line ok = gen_sctp:close(Sb), + ?line case recv_assoc_change(Sa, Loopback, Pb, Timeout) of + #sctp_assoc_change{state=comm_lost, + assoc_id=SaAssocId} -> ok; + timeout -> + ?line io:format("timeout waiting for comm_lost on Sa~n"), + ?line match_unless_solaris(ok, {timeout,flush()}) + end, ?line receive - {sctp,Sa,Loopback,Pb, - {[], - #sctp_assoc_change{state=comm_lost, - assoc_id=SaAssocId}}} -> ok - after Timeout -> - ?line test_server:fail({unexpected,flush()}) - end, - ?line receive - {sctp_error,Sa,enotconn} -> ok % Solaris - after 17 -> ok %% Only happens on Solaris - end, + {sctp_error,Sa,enotconn} -> ok % Solaris + after 17 -> ok + end, ?line ok = gen_sctp:close(Sa), %% ?line receive @@ -275,6 +301,30 @@ xfer_active(Config) when is_list(Config) -> end, ok. +recv_assoc_change(S, Addr, Port, Timeout) -> + receive + {sctp,S,Addr,Port,{[], #sctp_assoc_change{}=AssocChange}} -> + AssocChange; + {sctp,S,Addr,Port, + {[#sctp_sndrcvinfo{assoc_id=AssocId}], + #sctp_assoc_change{assoc_id=AssocId}=AssocChange}} -> + AssocChange + after Timeout -> + timeout + end. + +recv_paddr_change(S, Addr, Port, Timeout) -> + receive + {sctp,S,Addr,Port,{[], #sctp_paddr_change{}=PaddrChange}} -> + PaddrChange; + {sctp,S,Addr,Port, + {[#sctp_sndrcvinfo{assoc_id=AssocId}], + #sctp_paddr_change{assoc_id=AssocId}=PaddrChange}} -> + PaddrChange + after Timeout -> + timeout + end. + def_sndrcvinfo(doc) -> "Test that #sctp_sndrcvinfo{} parameters set on a socket " "are used by gen_sctp:send/4"; @@ -285,11 +335,11 @@ def_sndrcvinfo(Config) when is_list(Config) -> ?line Data = <<"What goes up, must come down.">>, %% ?line S1 = - ok(gen_sctp:open( + log_ok(gen_sctp:open( 0, [{sctp_default_send_param,#sctp_sndrcvinfo{ppid=17}}])), ?LOGVAR(S1), ?line P1 = - ok(inet:port(S1)), + log_ok(inet:port(S1)), ?LOGVAR(P1), ?line #sctp_sndrcvinfo{ppid=17, context=0, timetolive=0, assoc_id=0} = getopt(S1, sctp_default_send_param), @@ -297,10 +347,10 @@ def_sndrcvinfo(Config) when is_list(Config) -> gen_sctp:listen(S1, true), %% ?line S2 = - ok(gen_sctp:open()), + log_ok(gen_sctp:open()), ?LOGVAR(S2), ?line P2 = - ok(inet:port(S2)), + log_ok(inet:port(S2)), ?LOGVAR(P2), ?line #sctp_sndrcvinfo{ppid=0, context=0, timetolive=0, assoc_id=0} = getopt(S2, sctp_default_send_param), @@ -309,32 +359,57 @@ def_sndrcvinfo(Config) when is_list(Config) -> state=comm_up, error=0, assoc_id=S2AssocId} = S2AssocChange = - ok(gen_sctp:connect(S2, Loopback, P1, [])), + log_ok(gen_sctp:connect(S2, Loopback, P1, [])), ?LOGVAR(S2AssocChange), - ?line case ok(gen_sctp:recv(S1)) of - {Loopback, P2,[], + ?line case recv_event(log_ok(gen_sctp:recv(S1))) of + {Loopback,P2, #sctp_assoc_change{ + state=comm_up, + error=0, + assoc_id=S1AssocId}} -> + ?LOGVAR(S1AssocId); + {Loopback,P2, + #sctp_paddr_change{ + state=addr_confirmed, + error=0, + assoc_id=S1AssocId}} -> + ?LOGVAR(S1AssocId), + {Loopback,P2, + #sctp_assoc_change{ state=comm_up, error=0, - assoc_id=S1AssocId}} -> - ?LOGVAR(S1AssocId) + assoc_id=S1AssocId}} = + recv_event(log_ok(gen_sctp:recv(S1))) end, + ?line #sctp_sndrcvinfo{ - ppid=17, context=0, timetolive=0, assoc_id=S1AssocId} = + ppid=17, context=0, timetolive=0} = %, assoc_id=S1AssocId} = getopt( S1, sctp_default_send_param, #sctp_sndrcvinfo{assoc_id=S1AssocId}), ?line #sctp_sndrcvinfo{ - ppid=0, context=0, timetolive=0, assoc_id=S2AssocId} = + ppid=0, context=0, timetolive=0} = %, assoc_id=S2AssocId} = getopt( S2, sctp_default_send_param, #sctp_sndrcvinfo{assoc_id=S2AssocId}), %% ?line ok = gen_sctp:send(S1, S1AssocId, 1, <<"1: ",Data/binary>>), - ?line case ok(gen_sctp:recv(S2)) of + ?line case log_ok(gen_sctp:recv(S2)) of {Loopback,P1, [#sctp_sndrcvinfo{ stream=1, ppid=17, context=0, assoc_id=S2AssocId}], - <<"1: ",Data/binary>>} -> ok + <<"1: ",Data/binary>>} -> ok; + Event1 -> + ?line {Loopback,P1, + #sctp_paddr_change{state=addr_confirmed, + addr={_,P1}, + error=0, + assoc_id=S2AssocId}} = + recv_event(Event1), + ?line {Loopback,P1, + [#sctp_sndrcvinfo{ + stream=1, ppid=17, context=0, assoc_id=S2AssocId}], + <<"1: ",Data/binary>>} = + log_ok(gen_sctp:recv(S2)) end, %% ?line ok = @@ -354,7 +429,7 @@ def_sndrcvinfo(Config) when is_list(Config) -> %% ?line ok = gen_sctp:send(S1, S1AssocId, 0, <<"2: ",Data/binary>>), - ?line case ok(gen_sctp:recv(S2)) of + ?line case log_ok(gen_sctp:recv(S2)) of {Loopback,P1, [#sctp_sndrcvinfo{ stream=0, ppid=19, context=0, assoc_id=S2AssocId}], @@ -362,16 +437,18 @@ def_sndrcvinfo(Config) when is_list(Config) -> end, ?line ok = gen_sctp:send(S2, S2AssocChange, 1, <<"3: ",Data/binary>>), - ?line case ok(gen_sctp:recv(S1)) of + ?line case log_ok(gen_sctp:recv(S1)) of {Loopback,P2, [#sctp_sndrcvinfo{ stream=1, ppid=0, context=0, assoc_id=S1AssocId}], <<"3: ",Data/binary>>} -> ok; - {Loopback,P2,[], - #sctp_paddr_change{ - addr={Loopback,_}, state=addr_available, - error=0, assoc_id=S1AssocId}} -> - ?line case ok(gen_sctp:recv(S1)) of + Event2 -> + {Loopback,P2, + #sctp_paddr_change{ + addr={Loopback,_}, state=addr_available, + error=0, assoc_id=S1AssocId}} = + recv_event(Event2), + ?line case log_ok(gen_sctp:recv(S1)) of {Loopback,P2, [#sctp_sndrcvinfo{ stream=1, ppid=0, context=0, @@ -387,7 +464,7 @@ def_sndrcvinfo(Config) when is_list(Config) -> #sctp_sndrcvinfo{stream=0, ppid=20, assoc_id=S2AssocId}, <<"4: ",Data/binary>>) end), - ?line case ok(do_from_other_process(fun() -> gen_sctp:recv(S1) end)) of + ?line case log_ok(do_from_other_process(fun() -> gen_sctp:recv(S1) end)) of {Loopback,P2, [#sctp_sndrcvinfo{ stream=0, ppid=20, context=0, assoc_id=S1AssocId}], @@ -416,8 +493,12 @@ getopt(S, Opt, Param) -> setopt(S, Opt, Val) -> inet:setopts(S, [{Opt,Val}]). -ok({ok,X}) -> - io:format("OK: ~p~n", [X]), +log_ok(X) -> log(ok(X)). + +ok({ok,X}) -> X. + +log(X) -> + io:format("LOG[~w]: ~p~n", [self(),X]), X. flush() -> @@ -520,7 +601,10 @@ api_listen(Config) when is_list(Config) -> #sctp_assoc_change{ state=comm_lost}}} = gen_sctp:recv(Sa, infinity); - {error,#sctp_assoc_change{state=cant_assoc}} -> ok + {error,#sctp_assoc_change{state=cant_assoc}} -> + ok%; + %% {error,{Localhost,Pb,_,#sctp_assoc_change{state=cant_assoc}}} -> + %% ok end, ?line ok = gen_sctp:listen(Sb, true), ?line {ok,#sctp_assoc_change{state=comm_up, @@ -552,29 +636,41 @@ api_connect_init(Config) when is_list(Config) -> ?line {ok,Sa} = gen_sctp:open(), ?line case gen_sctp:connect_init(Sa, localhost, Pb, []) of {error,econnrefused} -> - ?line {ok,{Localhost, - Pb,[], - #sctp_assoc_change{state=comm_lost}}} = - gen_sctp:recv(Sa, infinity); + ?line {Localhost,Pb,#sctp_assoc_change{state=comm_lost}} = + recv_event(log_ok(gen_sctp:recv(Sa, infinity))); ok -> - ?line {ok,{Localhost, - Pb,[], - #sctp_assoc_change{state=cant_assoc}}} = - gen_sctp:recv(Sa, infinity) + ?line {Localhost,Pb,#sctp_assoc_change{state=cant_assoc}} = + recv_event(log_ok(gen_sctp:recv(Sa, infinity))) end, ?line ok = gen_sctp:listen(Sb, true), ?line case gen_sctp:connect_init(Sa, localhost, Pb, []) of ok -> - ?line {ok,{Localhost, - Pb,[], - #sctp_assoc_change{ - state = comm_up}}} = - gen_sctp:recv(Sa, infinity) + ?line {Localhost,Pb,#sctp_assoc_change{state=comm_up}} = + recv_event(log_ok(gen_sctp:recv(Sa, infinity))) end, ?line ok = gen_sctp:close(Sa), ?line ok = gen_sctp:close(Sb), ok. +recv_event({Addr,Port,[],#sctp_assoc_change{}=AssocChange}) -> + {Addr,Port,AssocChange}; +recv_event({Addr,Port, + [#sctp_sndrcvinfo{assoc_id=Assoc}], + #sctp_assoc_change{assoc_id=Assoc}=AssocChange}) -> + {Addr,Port,AssocChange}; +recv_event({Addr,Port,[],#sctp_paddr_change{}=PaddrChange}) -> + {Addr,Port,PaddrChange}; +recv_event({Addr,Port, + [#sctp_sndrcvinfo{assoc_id=Assoc}], + #sctp_paddr_change{assoc_id=Assoc}=PaddrChange}) -> + {Addr,Port,PaddrChange}; +recv_event({Addr,Port,[],#sctp_shutdown_event{}=ShutdownEvent}) -> + {Addr,Port,ShutdownEvent}; +recv_event({Addr,Port, + [#sctp_sndrcvinfo{assoc_id=Assoc}], + #sctp_shutdown_event{assoc_id=Assoc}=ShutdownEvent}) -> + {Addr,Port,ShutdownEvent}. + api_opts(doc) -> "Test socket options"; api_opts(suite) -> @@ -600,7 +696,7 @@ api_opts(Config) when is_list(Config) -> end. implicit_inet6(Config) when is_list(Config) -> - ?line Hostname = ok(inet:gethostname()), + ?line Hostname = log_ok(inet:gethostname()), ?line case gen_sctp:open(0, [inet6]) of {ok,S1} -> @@ -613,16 +709,16 @@ implicit_inet6(Config) when is_list(Config) -> ?line ok = gen_sctp:close(S1), %% ?line Localhost = - ok(inet:getaddr("localhost", inet6)), + log_ok(inet:getaddr("localhost", inet6)), ?line io:format("~s ~p~n", ["localhost",Localhost]), ?line S2 = - ok(gen_sctp:open(0, [{ip,Localhost}])), + log_ok(gen_sctp:open(0, [{ip,Localhost}])), ?line implicit_inet6(S2, Localhost), ?line ok = gen_sctp:close(S2), %% ?line io:format("~s ~p~n", [Hostname,Host]), ?line S3 = - ok(gen_sctp:open(0, [{ifaddr,Host}])), + log_ok(gen_sctp:open(0, [{ifaddr,Host}])), ?line implicit_inet6(S3, Host), ?line ok = gen_sctp:close(S1); {error,eafnosupport} -> @@ -635,25 +731,159 @@ implicit_inet6(Config) when is_list(Config) -> implicit_inet6(S1, Addr) -> ?line ok = gen_sctp:listen(S1, true), - ?line P1 = ok(inet:port(S1)), - ?line S2 = ok(gen_sctp:open(0, [inet6])), - ?line P2 = ok(inet:port(S2)), + ?line P1 = log_ok(inet:port(S1)), + ?line S2 = log_ok(gen_sctp:open(0, [inet6])), + ?line P2 = log_ok(inet:port(S2)), ?line #sctp_assoc_change{state=comm_up} = - ok(gen_sctp:connect(S2, Addr, P1, [])), - ?line case ok(gen_sctp:recv(S1)) of - {Addr,P2,[],#sctp_assoc_change{state=comm_up}} -> - ok + log_ok(gen_sctp:connect(S2, Addr, P1, [])), + ?line case recv_event(log_ok(gen_sctp:recv(S1))) of + {Addr,P2,#sctp_assoc_change{state=comm_up}} -> + ok; + {Addr,P2,#sctp_paddr_change{state=addr_confirmed, + addr={Addr,P2}, + error=0}} -> + {Addr,P2,#sctp_assoc_change{state=comm_up}} = + recv_event(log_ok(gen_sctp:recv(S1))) end, - ?line case ok(inet:sockname(S1)) of + ?line case log_ok(inet:sockname(S1)) of {Addr,P1} -> ok; {{0,0,0,0,0,0,0,0},P1} -> ok end, - ?line case ok(inet:sockname(S2)) of + ?line case log_ok(inet:sockname(S2)) of {Addr,P2} -> ok; {{0,0,0,0,0,0,0,0},P2} -> ok end, ?line ok = gen_sctp:close(S2). +basic_stream(doc) -> + "Hello world stream socket"; +basic_stream(suite) -> + []; +basic_stream(Config) when is_list(Config) -> + ?line {ok,S} = gen_sctp:open([{type,stream}]), + ?line ok = gen_sctp:listen(S, true), + ?line ok = + do_from_other_process( + fun () -> gen_sctp:listen(S, 10) end), + ?line ok = gen_sctp:close(S), + ok. + +xfer_stream_min(doc) -> + "Minimal data transfer"; +xfer_stream_min(suite) -> + []; +xfer_stream_min(Config) when is_list(Config) -> + ?line Stream = 0, + ?line Data = <<"The quick brown fox jumps over a lazy dog 0123456789">>, + ?line Loopback = {127,0,0,1}, + ?line {ok,Sb} = gen_sctp:open([{type,seqpacket}]), + ?line ?LOGVAR(Sb), + ?line {ok,Pb} = inet:port(Sb), + ?line ?LOGVAR(Pb), + ?line ok = gen_sctp:listen(Sb, true), + + ?line {ok,Sa} = gen_sctp:open([{type,stream}]), + ?line ?LOGVAR(Sa), + ?line {ok,Pa} = inet:port(Sa), + ?line ?LOGVAR(Pa), + ?line #sctp_assoc_change{state=comm_up, + error=0, + outbound_streams=SaOutboundStreams, + inbound_streams=SaInboundStreams, + assoc_id=SaAssocId_X} = + log_ok(gen_sctp:connect(Sa, Loopback, Pb, [])), + ?line ?LOGVAR(SaAssocId_X), + ?line [{_,#sctp_paddrinfo{assoc_id=SaAssocId,state=active}}] = + log_ok(inet:getopts(Sa, [{sctp_get_peer_addr_info, + #sctp_paddrinfo{address={Loopback,Pb}}}])), + ?line ?LOGVAR(SaAssocId), + ?line match_unless_solaris(SaAssocId_X, SaAssocId), + + ?line {SbOutboundStreams,SbInboundStreams,SbAssocId} = + case recv_event(log_ok(gen_sctp:recv(Sb, infinity))) of + {Loopback,Pa, + #sctp_assoc_change{state=comm_up, + error=0, + outbound_streams=OS, + inbound_streams=IS, + assoc_id=AI}} -> + {OS,IS,AI}; + {Loopback,Pa, + #sctp_paddr_change{state=addr_confirmed, + addr={Loopback,Pa}, + error=0, + assoc_id=AI}} -> + {Loopback,Pa, + ?line #sctp_assoc_change{state=comm_up, + error=0, + outbound_streams=OS, + inbound_streams=IS, + assoc_id=AI}} = + recv_event(log_ok(gen_sctp:recv(Sb, infinity))), + {OS,IS,AI} + end, + ?line ?LOGVAR(SbAssocId), + ?line SaOutboundStreams = SbInboundStreams, + ?line ?LOGVAR(SaOutboundStreams), + ?line SbOutboundStreams = SaInboundStreams, + ?line ?LOGVAR(SbOutboundStreams), + ?line ok = gen_sctp:send(Sa, SaAssocId, 0, Data), + ?line case gen_sctp:recv(Sb, infinity) of + {ok,{Loopback, + Pa, + [#sctp_sndrcvinfo{stream=Stream, + assoc_id=SbAssocId}], + Data}} -> ok; + {ok,{Loopback, + Pa,[], + #sctp_paddr_change{addr = {Loopback,_}, + state = addr_available, + error = 0, + assoc_id = SbAssocId}}} -> + {ok,{Loopback, + Pa, + [#sctp_sndrcvinfo{stream=Stream, + assoc_id=SbAssocId}], + Data}} = gen_sctp:recv(Sb, infinity) + end, + ?line ok = + do_from_other_process( + fun () -> gen_sctp:send(Sb, SbAssocId, 0, Data) end), + ?line case log_ok(gen_sctp:recv(Sa, infinity)) of + {Loopback,Pb, + [#sctp_sndrcvinfo{stream=Stream, + assoc_id=SaAssocId}], + Data} -> ok; + Event1 -> + ?line {Loopback,Pb, + #sctp_paddr_change{state=addr_confirmed, + addr={_,Pb}, + error=0, + assoc_id=SaAssocId}} = + recv_event(Event1), + ?line {Loopback,Pb, + [#sctp_sndrcvinfo{stream=Stream, + assoc_id=SaAssocId}], + Data} = + log_ok(gen_sctp:recv(Sa, infinity)) + end, + ?line ok = gen_sctp:close(Sa), + ?line {Loopback,Pa, + #sctp_shutdown_event{assoc_id=SbAssocId}} = + recv_event(log_ok(gen_sctp:recv(Sb, infinity))), + ?line {Loopback,Pa, + #sctp_assoc_change{state=shutdown_comp, + error=0, + assoc_id=SbAssocId}} = + recv_event(log_ok(gen_sctp:recv(Sb, infinity))), + ?line ok = gen_sctp:close(Sb), + + ?line receive + Msg -> test_server:fail({received,Msg}) + after 17 -> ok + end, + ok. + do_from_other_process(Fun) -> @@ -681,3 +911,419 @@ do_from_other_process(Fun) -> {'DOWN',Mref,_,_,Reason} -> erlang:exit(Reason) end. + + + +peeloff(doc) -> + "Peel off an SCTP stream socket"; +peeloff(suite) -> + []; +peeloff(Config) when is_list(Config) -> + ?line Addr = {127,0,0,1}, + ?line Stream = 0, + ?line Timeout = 333, + ?line S1 = socket_open([{ifaddr,Addr}], Timeout), + ?line ?LOGVAR(S1), + ?line P1 = socket_call(S1, get_port), + ?line ?LOGVAR(P1), + ?line Socket1 = socket_call(S1, get_socket), + ?line ?LOGVAR(Socket1), + ?line socket_call(S1, {listen,true}), + ?line S2 = socket_open([{ifaddr,Addr}], Timeout), + ?line ?LOGVAR(S2), + ?line P2 = socket_call(S2, get_port), + ?line ?LOGVAR(P2), + ?line Socket2 = socket_call(S2, get_socket), + ?line ?LOGVAR(Socket2), + %% + ?line socket_call(S2, {connect_init,Addr,P1,[]}), + ?line S2Ai = + receive + {S2,{Addr,P1, + #sctp_assoc_change{ + state=comm_up, + assoc_id=AssocId2}}} -> AssocId2 + after Timeout -> + socket_bailout([S1,S2]) + end, + ?line ?LOGVAR(S2Ai), + ?line S1Ai = + receive + {S1,{Addr,P2, + #sctp_assoc_change{ + state=comm_up, + assoc_id=AssocId1}}} -> AssocId1 + after Timeout -> + socket_bailout([S1,S2]) + end, + ?line ?LOGVAR(S1Ai), + %% + ?line socket_call(S2, {send,S2Ai,Stream,<<"Number one">>}), + ?line + receive + {S1,{Addr,P2,S1Ai,Stream,<<"Number one">>}} -> ok + after Timeout -> + socket_bailout([S1,S2]) + end, + ?line socket_call(S2, {send,Socket1,S1Ai,Stream,<<"Number two">>}), + ?line + receive + {S2,{Addr,P1,S2Ai,Stream,<<"Number two">>}} -> ok + after Timeout -> + socket_bailout([S1,S2]) + end, + %% + ?line S3 = socket_peeloff(Socket1, S1Ai, Timeout), + ?line ?LOGVAR(S3), + ?line P3_X = socket_call(S3, get_port), + ?line ?LOGVAR(P3_X), + ?line P3 = case P3_X of 0 -> P1; _ -> P3_X end, + ?line [{_,#sctp_paddrinfo{assoc_id=S3Ai,state=active}}] = + socket_call(S3, + {getopts,[{sctp_get_peer_addr_info, + #sctp_paddrinfo{address={Addr,P2}}}]}), + %%?line S3Ai = S1Ai, + ?line ?LOGVAR(S3Ai), + %% + ?line socket_call(S3, {send,S3Ai,Stream,<<"Number three">>}), + ?line + receive + {S2,{Addr,P3,S2Ai,Stream,<<"Number three">>}} -> ok + after Timeout -> + socket_bailout([S1,S2,S3]) + end, + ?line socket_call(S3, {send,Socket2,S2Ai,Stream,<<"Number four">>}), + ?line + receive + {S3,{Addr,P2,S3Ai,Stream,<<"Number four">>}} -> ok + after Timeout -> + socket_bailout([S1,S2,S3]) + end, + %% + ?line inet:i(sctp), + ?line socket_close_verbose(S1), + ?line socket_close_verbose(S2), + ?line + receive + {S3,{Addr,P2,#sctp_shutdown_event{assoc_id=S3Ai_X}}} -> + ?line match_unless_solaris(S3Ai, S3Ai_X) + after Timeout -> + socket_bailout([S3]) + end, + ?line + receive + {S3,{Addr,P2,#sctp_assoc_change{state=shutdown_comp, + assoc_id=S3Ai}}} -> ok + after Timeout -> + socket_bailout([S3]) + end, + ?line socket_close_verbose(S3), + ?line [] = flush(), + ok. + + + +buffers(doc) -> + ["Check sndbuf and recbuf behaviour"]; +buffers(suite) -> + []; +buffers(Config) when is_list(Config) -> + ?line Limit = 4096, + ?line Addr = {127,0,0,1}, + ?line Stream = 1, + ?line Timeout = 3333, + ?line S1 = socket_open([{ip,Addr}], Timeout), + ?line ?LOGVAR(S1), + ?line P1 = socket_call(S1, get_port), + ?line ?LOGVAR(P1), + ?line ok = socket_call(S1, {listen,true}), + ?line S2 = socket_open([{ip,Addr}], Timeout), + ?line ?LOGVAR(S2), + ?line P2 = socket_call(S2, get_port), + ?line ?LOGVAR(P2), + %% + ?line socket_call(S2, {connect_init,Addr,P1,[]}), + ?line S2Ai = + receive + {S2,{Addr,P1, + #sctp_assoc_change{ + state=comm_up, + assoc_id=AssocId2}}} -> AssocId2 + after Timeout -> + socket_bailout([S1,S2]) + end, + ?line S1Ai = + receive + {S1,{Addr,P2, + #sctp_assoc_change{ + state=comm_up, + assoc_id=AssocId1}}} -> AssocId1 + after Timeout -> + socket_bailout([S1,S2]) + end, + %% + ?line socket_call(S1, {setopts,[{recbuf,Limit}]}), + ?line Recbuf = + case socket_call(S1, {getopts,[recbuf]}) of + [{recbuf,RB1}] when RB1 >= Limit -> RB1 + end, + ?line Data = mk_data(Recbuf+Limit), + ?line socket_call(S2, {setopts,[{sndbuf,Recbuf+Limit}]}), + ?line socket_call(S2, {send,S2Ai,Stream,Data}), + ?line + receive + {S1,{Addr,P2,S1Ai,Stream,Data}} -> ok + after Timeout -> + socket_bailout([S1,S2]) + end, + %% + ?line socket_close_verbose(S1), + ?line + receive + {S2,{Addr,P1,#sctp_shutdown_event{assoc_id=S2Ai}}} -> ok + after Timeout -> + socket_bailout([S2]) + end, + ?line + receive + {S2,{Addr,P1,#sctp_assoc_change{state=shutdown_comp, + assoc_id=S2Ai}}} -> ok + after Timeout -> + socket_bailout([S2]) + end, + ?line socket_close_verbose(S2), + ?line [] = flush(), + ok. + +mk_data(Bytes) -> + mk_data(0, Bytes, <<>>). +%% +mk_data(N, Bytes, Bin) when N < Bytes -> + mk_data(N+4, Bytes, <<Bin/binary,N:32>>); +mk_data(_, _, Bin) -> + Bin. + +%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% socket gen_server ultra light + +socket_open(SocketOpts, Timeout) -> + Opts = [{type,seqpacket},{active,once},binary|SocketOpts], + Starter = + fun () -> + {ok,Socket} = + gen_sctp:open(Opts), + Socket + end, + s_start(Starter, Timeout). + +socket_peeloff(Socket, AssocId, Timeout) -> + Opts = [{active,once},binary], + Starter = + fun () -> + {ok,NewSocket} = + gen_sctp:peeloff(Socket, AssocId), + ok = inet:setopts(NewSocket, Opts), + NewSocket + end, + s_start(Starter, Timeout). + +socket_close_verbose(S) -> + History = socket_history(socket_close(S)), + io:format("socket_close ~p:~n ~p.~n", [S,History]), + History. + +socket_close(S) -> + s_req(S, close). + +socket_call(S, Request) -> + s_req(S, {Request}). + +%% socket_get(S, Key) -> +%% s_req(S, {get,Key}). + +socket_bailout([S|Ss]) -> + History = socket_history(socket_close(S)), + io:format("bailout ~p:~n ~p.~n", [S,History]), + socket_bailout(Ss); +socket_bailout([]) -> + io:format("flush: ~p.~n", [flush()]), + test_server:fail(socket_bailout). + +socket_history({State,Flush}) -> + {lists:keysort( + 2, + lists:flatten( + [[{Key,Val} || Val <- Vals] + || {Key,Vals} <- gb_trees:to_list(State)])), + Flush}. + +s_handler(Socket) -> + fun ({listen,Listen}) -> + ok = gen_sctp:listen(Socket, Listen); + (get_port) -> + ok(inet:port(Socket)); + (get_socket) -> + Socket; + ({connect_init,ConAddr,ConPort,ConOpts}) -> + ok = gen_sctp:connect_init(Socket, ConAddr, ConPort, ConOpts); + ({send,AssocId,Stream,Data}) -> + ok = gen_sctp:send(Socket, AssocId, Stream, Data); + ({send,OtherSocket,AssocId,Stream,Data}) -> + ok = gen_sctp:send(OtherSocket, AssocId, Stream, Data); + ({setopts,Opts}) -> + ok = inet:setopts(Socket, Opts); + ({getopts,Optnames}) -> + ok(inet:getopts(Socket, Optnames)) + end. + +s_req(S, Req) -> + Mref = erlang:monitor(process, S), + S ! {self(),Mref,Req}, + receive + {'DOWN',Mref,_,_,Error} -> + exit(Error); + {S,Mref,Reply} -> + erlang:demonitor(Mref), + receive {'DOWN',Mref,_,_,_} -> ok after 0 -> ok end, + Reply + end. + +s_start(Starter, Timeout) -> + Parent = self(), + Owner = + spawn_link( + fun () -> + s_start(Starter(), Timeout, Parent) + end), + Owner. + +s_start(Socket, Timeout, Parent) -> + Handler = s_handler(Socket), + try + s_loop(Socket, Timeout, Parent, Handler, gb_trees:empty()) + catch + Class:Reason -> + Stacktrace = erlang:get_stacktrace(), + io:format(?MODULE_STRING":socket exception ~w:~w at~n" + "~p.~n", [Class,Reason,Stacktrace]), + erlang:raise(Class, Reason, Stacktrace) + end. + +s_loop(Socket, Timeout, Parent, Handler, State) -> + receive + {Parent,Ref,close} -> % socket_close() + erlang:send_after(Timeout, self(), {Parent,Ref,exit}), + s_loop(Socket, Timeout, Parent, Handler, State); + {Parent,Ref,exit} -> + ok = gen_sctp:close(Socket), + Key = exit, + Val = {now(),Socket}, + NewState = gb_push(Key, Val, State), + Parent ! {self(),Ref,{NewState,flush()}}; + {Parent,Ref,{Msg}} -> + Result = Handler(Msg), + Key = req, + Val = {now(),{Msg,Result}}, + NewState = gb_push(Key, Val, State), + Parent ! {self(),Ref,Result}, + s_loop(Socket, Timeout, Parent, Handler, NewState); + %% {Parent,Ref,{get,Key}} -> + %% Parent ! {self(),Ref,gb_get(Key, State)}, + %% s_loop(Socket, Timeout, Parent, Handler, State); + {sctp,Socket,Addr,Port, + {[#sctp_sndrcvinfo{stream=Stream,assoc_id=AssocId}=SRI],Data}} + when not is_tuple(Data) -> + case gb_get({assoc_change,AssocId}, State) of + [{_,{Addr,Port, + #sctp_assoc_change{ + state=comm_up, + inbound_streams=Is}}}|_] + when 0 =< Stream, Stream < Is-> ok; + [] -> ok + end, + Key = {msg,AssocId,Stream}, + Val = {now(),{Addr,Port,SRI,Data}}, + NewState = gb_push(Key, Val, State), + Parent ! {self(),{Addr,Port,AssocId,Stream,Data}}, + again(Socket), + s_loop(Socket, Timeout, Parent, Handler, NewState); + {sctp,Socket,Addr,Port, + {SRI,#sctp_assoc_change{assoc_id=AssocId,state=St}=SAC}} -> + case SRI of + [#sctp_sndrcvinfo{assoc_id=AssocId,stream=0}] -> ok; + [] -> ok + end, + Key = {assoc_change,AssocId}, + Val = {now(),{Addr,Port,SAC}}, + case {gb_get(Key, State),St} of + {[],_} -> ok; + {[{_,{Addr,Port,#sctp_assoc_change{state=comm_up}}}|_],_} + when St =:= comm_lost; St =:= shutdown_comp -> ok + end, + NewState = gb_push(Key, Val, State), + Parent ! {self(),{Addr,Port,SAC}}, + again(Socket), + s_loop(Socket, Timeout, Parent, Handler, NewState); + {sctp,Socket,Addr,Port, + {SRI,#sctp_paddr_change{assoc_id=AssocId, + addr={_,P}, + state=St}=SPC}} -> + match_unless_solaris(Port, P), + case SRI of + [#sctp_sndrcvinfo{assoc_id=AssocId,stream=0}] -> ok; + [] -> ok + end, + case {gb_get({assoc_change,AssocId}, State),St} of + {[{_,{Addr,Port,#sctp_assoc_change{state=comm_up}}}|_], + addr_available} -> ok; + {[],addr_confirmed} -> ok + end, + Key = {paddr_change,AssocId}, + Val = {now(),{Addr,Port,SPC}}, + NewState = gb_push(Key, Val, State), + again(Socket), + s_loop(Socket, Timeout, Parent, Handler, NewState); + {sctp,Socket,Addr,Port, + {SRI,#sctp_shutdown_event{assoc_id=AssocId}=SSE}} -> + case SRI of + [#sctp_sndrcvinfo{assoc_id=AssocId,stream=0}] -> ok; + [] -> ok + end, + case gb_get({assoc_change,AssocId}, State) of + [{_,{Addr,Port,#sctp_assoc_change{state=comm_up}}}|_] -> ok; + [] -> ok + end, + Key = {shutdown_event,AssocId}, + Val = {now(),{Addr,Port}}, + NewState = gb_push(Key, Val, State), + Parent ! {self(), {Addr,Port,SSE}}, + again(Socket), + s_loop(Socket, Timeout, Parent, Handler, NewState); + Unexpected -> + erlang:error({unexpected,Unexpected}) + end. + +again(Socket) -> + inet:setopts(Socket, [{active,once}]). + +gb_push(Key, Val, GBT) -> + case gb_trees:lookup(Key, GBT) of + none -> + gb_trees:insert(Key, [Val], GBT); + {value,V} -> + gb_trees:update(Key, [Val|V], GBT) + end. + +gb_get(Key, GBT) -> + case gb_trees:lookup(Key, GBT) of + none -> + []; + {value,V} -> + V + end. + +match_unless_solaris(A, B) -> + case os:type() of + {unix,sunos} -> B; + _ -> A = B + end. diff --git a/lib/kernel/test/global_SUITE.erl b/lib/kernel/test/global_SUITE.erl index 1e7bcf1766..60035b50a0 100644 --- a/lib/kernel/test/global_SUITE.erl +++ b/lib/kernel/test/global_SUITE.erl @@ -436,7 +436,7 @@ lock_global2(Id, Parent) -> %cp1 - cp3 are started, and the name 'test' registered for a process on %test_server. Then it is checked that the name is registered on all -%nodes, using whereis_name and safe_whereis_name. Check that the same +%nodes, using whereis_name. Check that the same %name can't be registered with another value. Exit the registered %process and check that the name disappears. Register a new process %(Pid2) under the name 'test'. Let another new process (Pid3) @@ -465,10 +465,6 @@ names(Config) when is_list(Config) -> % test that it is registered at all nodes ?line ?UNTIL(begin - (Pid =:= global:safe_whereis_name(test)) and - (Pid =:= rpc:call(Cp1, global, safe_whereis_name, [test])) and - (Pid =:= rpc:call(Cp2, global, safe_whereis_name, [test])) and - (Pid =:= rpc:call(Cp3, global, safe_whereis_name, [test])) and (Pid =:= global:whereis_name(test)) and (Pid =:= rpc:call(Cp1, global, whereis_name, [test])) and (Pid =:= rpc:call(Cp2, global, whereis_name, [test])) and @@ -566,10 +562,7 @@ names_hidden(Config) when is_list(Config) -> % Check that it didn't get registered on visible nodes ?line - ?UNTIL((undefined =:= global:safe_whereis_name(test)) and - (undefined =:= rpc:call(Cp1, global, safe_whereis_name, [test])) and - (undefined =:= rpc:call(Cp2, global, safe_whereis_name, [test])) and - (undefined =:= global:whereis_name(test)) and + ?UNTIL((undefined =:= global:whereis_name(test)) and (undefined =:= rpc:call(Cp1, global, whereis_name, [test])) and (undefined =:= rpc:call(Cp2, global, whereis_name, [test]))), @@ -579,11 +572,7 @@ names_hidden(Config) when is_list(Config) -> % test that it is registered at all nodes ?line - ?UNTIL((Pid =:= global:safe_whereis_name(test)) and - (Pid =:= rpc:call(Cp1, global, safe_whereis_name, [test])) and - (Pid =:= rpc:call(Cp2, global, safe_whereis_name, [test])) and - (HPid =:= rpc:call(Cp3, global, safe_whereis_name, [test])) and - (Pid =:= global:whereis_name(test)) and + ?UNTIL((Pid =:= global:whereis_name(test)) and (Pid =:= rpc:call(Cp1, global, whereis_name, [test])) and (Pid =:= rpc:call(Cp2, global, whereis_name, [test])) and (HPid =:= rpc:call(Cp3, global, whereis_name, [test])) and diff --git a/lib/kernel/test/inet_res_SUITE.erl b/lib/kernel/test/inet_res_SUITE.erl index 8a3d220e46..15b0ed5718 100644 --- a/lib/kernel/test/inet_res_SUITE.erl +++ b/lib/kernel/test/inet_res_SUITE.erl @@ -136,21 +136,22 @@ ns_init(ZoneDir, PrivDir, DataDir) -> atom_to_list(ZoneDir)]}, stderr_to_stdout, eof]), - ns_start(ZoneDir, NS, P); + ns_start(ZoneDir, PrivDir, NS, P); _ -> throw("Only run on Unix") end. -ns_start(ZoneDir, NS, P) -> +ns_start(ZoneDir, PrivDir, NS, P) -> case ns_collect(P) of eof -> erlang:error(eof); "Running: "++_ -> {ZoneDir,NS,P}; "Error: "++Error -> + ns_printlog(filename:join([PrivDir,ZoneDir,"named.log"])), throw(Error); _ -> - ns_start(ZoneDir, NS, P) + ns_start(ZoneDir, PrivDir, NS, P) end. ns_end(undefined, _PrivDir) -> undefined; diff --git a/lib/kernel/test/inet_res_SUITE_data/run-named b/lib/kernel/test/inet_res_SUITE_data/run-named index 7caa3756ef..39e7b1d5aa 100755 --- a/lib/kernel/test/inet_res_SUITE_data/run-named +++ b/lib/kernel/test/inet_res_SUITE_data/run-named @@ -47,6 +47,7 @@ CONF_FILE=named.conf INC_FILE=named_inc.conf PID_FILE=named.pid LOG_FILE=named.log +EXIT_FILE=named.exit error () { r=$? @@ -71,10 +72,14 @@ test -d "$SRCDIR" || \ test -f "$SRCDIR/$INC_FILE" || \ error "Missing file: $SRCDIR/$INC_FILE !" -# Locate named and check version +# Locate named and check version. +# The bind-named name is used for tricking Apparmor and such +# by copying/hardlinking the real named to that name. NAMED=named -for n in /usr/sbin/named /usr/sbin/in.named; do - test -x "$n" && NAMED="$n" +for n in /usr/local/bin/bind-named /usr/local/bin/named \ + /usr/sbin/bind-named /usr/sbin/named /usr/sbin/in.named +do + test -x "$n" && NAMED="$n" && break done NAMED_VER="`"$NAMED" -v 2>&1`" || \ error "Name server not found!" @@ -145,19 +150,27 @@ cat >>"$CONF_FILE" <<-CONF_FILE ( cd "$SRCDIR" && ls -1 ) | while read f; do cp -fp "$SRCDIR/$f" . done +rm -f "$EXIT_FILE" # Start nameserver echo "Cwd: `pwd`" echo "Nameserver: $NAMED_VER" echo "Port: $2" echo "ZoneDir: $3" -$NAMED $NAMED_FG -c "$CONF_FILE" >"$LOG_FILE" 2>&1 </dev/null & -NAMED=$! -trap "kill -TERM $NAMED >/dev/null 2>&1; wait $NAMED >/dev/null 2>&1" \ +echo "Command: $NAMED $NAMED_FG -c $CONF_FILE" +($NAMED $NAMED_FG -c "$CONF_FILE" >"$LOG_FILE" 2>&1 </dev/null; \ + echo "$?" >"$EXIT_FILE")& +NAMED_PID=$! +trap "kill -TERM $NAMED_PID >/dev/null 2>&1; wait $NAMED_PID >/dev/null 2>&1" \ 0 1 2 3 15 sleep 2 # Give name server time to load its zone files -echo "Running: Enter \`\`quit'' to terminate nameserver[$NAMED]..." -while read LINE; do - test :"$LINE" = :'quit' && break -done +if [ -f "$EXIT_FILE" ]; then + ERROR="`cat "$EXIT_FILE"`" + (exit "$ERROR")& error "$NAMED returned $ERROR on start" +else + echo "Running: Enter \`\`quit'' to terminate nameserver[$NAMED_PID]..." + while read LINE; do + test :"$LINE" = :'quit' && break + done +fi echo "Closing: Terminating nameserver..." diff --git a/lib/megaco/doc/src/notes.xml b/lib/megaco/doc/src/notes.xml index dfbc5895d1..baf8c6a201 100644 --- a/lib/megaco/doc/src/notes.xml +++ b/lib/megaco/doc/src/notes.xml @@ -55,6 +55,13 @@ </item> <item> + <p>ASN.1 no longer makes use of a driver to accelerate encode/decode, + instead it uses NIFs. The encoding config option is <em>still</em> + the same, i.e. <c>driver</c>. </p> + <p>Own Id: OTP-9672</p> + </item> + + <item> <p>The profiling test tool has been rewritten. </p> <p>Håkan Mattsson</p> <p>Own Id: OTP-9679</p> diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 8ef573a948..1bb80f8fe3 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -38,7 +38,35 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.5</title> + <section><title>Mnesia 4.5.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix deadlock in mnesia:del_table_copy/2.</p> + <p> + Own Id: OTP-9689 Aux Id: seq11927 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Allow schema operations when using different mnesia + versions.</p> + <p> + Own Id: OTP-9657 Aux Id: seq11926 </p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/mnesia/src/mnesia.appup.src b/lib/mnesia/src/mnesia.appup.src index fe4e5e2e7a..e0954ad206 100644 --- a/lib/mnesia/src/mnesia.appup.src +++ b/lib/mnesia/src/mnesia.appup.src @@ -1,12 +1,14 @@ %% -*- erlang -*- -{"%VSN%", +{"%VSN%", [ + {"4.5", [{restart_application, mnesia}]}, {"4.4.19", [{restart_application, mnesia}]}, {"4.4.18", [{restart_application, mnesia}]}, {"4.4.17", [{restart_application, mnesia}]}, {"4.4.16", [{restart_application, mnesia}]} ], [ + {"4.5", [{restart_application, mnesia}]}, {"4.4.19", [{restart_application, mnesia}]}, {"4.4.18", [{restart_application, mnesia}]}, {"4.4.17", [{restart_application, mnesia}]}, diff --git a/lib/mnesia/src/mnesia_bup.erl b/lib/mnesia/src/mnesia_bup.erl index 47dcdad7ac..14414537b9 100644 --- a/lib/mnesia/src/mnesia_bup.erl +++ b/lib/mnesia/src/mnesia_bup.erl @@ -372,7 +372,9 @@ mk_str() -> lists:concat([node()] ++ Now ++ ".TMP"). make_initial_backup(Ns, Opaque, Mod) -> - Schema = [{schema, schema, mnesia_schema:get_initial_schema(disc_copies, Ns)}], + Orig = mnesia_schema:get_initial_schema(disc_copies, Ns), + Modded = proplists:delete(storage_properties, proplists:delete(majority, Orig)), + Schema = [{schema, schema, Modded}], O2 = do_apply(Mod, open_write, [Opaque], Opaque), O3 = do_apply(Mod, write, [O2, [mnesia_log:backup_log_header()]], O2), O4 = do_apply(Mod, write, [O3, Schema], O3), diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 1d3bd55b48..6a561394d5 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -289,40 +289,7 @@ get_remote_cstructs() -> get_cstructs() -> {cstructs, Cstructs, Running} = call(get_cstructs), Node = node(group_leader()), - {cstructs, normalize_cstructs(Cstructs, Node), Running}. - -normalize_cstructs(Cstructs, Node) -> - %% backward-compatibility hack; normalize before returning - case rpc:call(Node, mnesia_lib, val, [{schema,cstruct}]) of - {badrpc, _} -> - %% assume it's not a schema merge - Cstructs; - #cstruct{} -> - %% same format - Cstructs; - Cstruct -> - %% some other format - RemoteFields = [F || {F,_} <- rpc:call(Node, mnesia_schema, cs2list, [Cstruct])], - [convert_cs(Cs, RemoteFields) || Cs <- Cstructs] - end. - -convert_cs(Cs, Fields) -> - MyFields = record_info(fields, cstruct), - convert(tl(tuple_to_list(Cs)), MyFields, Fields, []). - -convert([H|T], [F|FsL], [F|FsR], Acc) -> - convert(T, FsL, FsR, [H|Acc]); -convert([H|T], [Fl|FsL] = L, [Fr|FsR] = R, Acc) -> - case {lists:member(Fl, FsR), lists:member(Fr, FsL)} of - {true, false} -> - convert(T, L, FsR, [H|Acc]); - {false, true} -> - %% Field Fl doesn't exist on receiver side; skip. - convert(T, FsL, R, Acc) - end; -convert([], _, _, Acc) -> - list_to_tuple([cstruct|lists:reverse(Acc)]). - + {cstructs, mnesia_schema:normalize_cs(Cstructs, Node), Running}. update(Fun) -> call({update,Fun}). diff --git a/lib/mnesia/src/mnesia_event.erl b/lib/mnesia/src/mnesia_event.erl index ec6b99ecaa..5a060a28ff 100644 --- a/lib/mnesia/src/mnesia_event.erl +++ b/lib/mnesia/src/mnesia_event.erl @@ -121,7 +121,7 @@ handle_system_event({mnesia_up, Node}, State) -> {ok, State#state{nodes = Nodes}}; handle_system_event({mnesia_down, Node}, State) -> - case mnesia:system_info(fallback_activated) of + case mnesia:system_info(fallback_activated) andalso Node =/= node() of true -> case mnesia_monitor:get_env(fallback_error_function) of {mnesia, lkill} -> @@ -129,8 +129,8 @@ handle_system_event({mnesia_down, Node}, State) -> "must be restarted. Forcing shutdown " "after mnesia_down from ~p...~n", report_fatal(Msg, [Node], nocore, State#state.dumped_core), - mnesia:lkill(), - exit(fatal); + catch exit(whereis(mnesia_monitor), fatal), + {ok, State}; {UserMod, UserFunc} -> Msg = "Warning: A fallback is installed and Mnesia got mnesia_down " "from ~p. ~n", diff --git a/lib/mnesia/src/mnesia_frag.erl b/lib/mnesia/src/mnesia_frag.erl index 9e77fe0b9f..4a1616e054 100644 --- a/lib/mnesia/src/mnesia_frag.erl +++ b/lib/mnesia/src/mnesia_frag.erl @@ -758,7 +758,7 @@ make_activate(Tab, Props) -> [] -> Cs2 = Cs#cstruct{frag_properties = Props}, [Cs3] = expand_cstruct(Cs2, activate), - TabDef = mnesia_schema:cs2list(Cs3), + TabDef = mnesia_schema:vsn_cs2list(Cs3), Op = {op, change_table_frag, activate, TabDef}, [[Op]]; BadProps -> @@ -783,7 +783,7 @@ make_deactivate(Tab) -> mnesia:abort({combine_error, Tab, "Too many fragments"}); true -> Cs2 = Cs#cstruct{frag_properties = []}, - TabDef = mnesia_schema:cs2list(Cs2), + TabDef = mnesia_schema:vsn_cs2list(Cs2), Op = {op, change_table_frag, deactivate, TabDef}, [[Op]] end. @@ -850,7 +850,7 @@ make_add_frag(Tab, SortedNs) -> SplitOps = split(Tab, FH2, FromIndecies, FragNames, []), Cs2 = replace_frag_hash(Cs, FH2), - TabDef = mnesia_schema:cs2list(Cs2), + TabDef = mnesia_schema:vsn_cs2list(Cs2), BaseOp = {op, change_table_frag, {add_frag, SortedNs}, TabDef}, [BaseOp, NewOp | SplitOps]. @@ -962,7 +962,7 @@ make_del_frag(Tab) -> LastFrag = element(N, FragNames), [LastOp] = mnesia_schema:make_delete_table(LastFrag, single_frag), Cs2 = replace_frag_hash(Cs, FH2), - TabDef = mnesia_schema:cs2list(Cs2), + TabDef = mnesia_schema:vsn_cs2list(Cs2), BaseOp = {op, change_table_frag, del_frag, TabDef}, [BaseOp, LastOp | MergeOps]; _ -> @@ -1075,7 +1075,7 @@ make_add_node(Tab, Node) when is_atom(Node) -> Props = Cs#cstruct.frag_properties, Props2 = lists:keyreplace(node_pool, 1, Props, {node_pool, Pool2}), Cs2 = Cs#cstruct{frag_properties = Props2}, - TabDef = mnesia_schema:cs2list(Cs2), + TabDef = mnesia_schema:vsn_cs2list(Cs2), Op = {op, change_table_frag, {add_node, Node}, TabDef}, [Op]; true -> @@ -1104,7 +1104,7 @@ make_del_node(Tab, Node) when is_atom(Node) -> Pool2 = Pool -- [Node], Props = lists:keyreplace(node_pool, 1, Cs#cstruct.frag_properties, {node_pool, Pool2}), Cs2 = Cs#cstruct{frag_properties = Props}, - TabDef = mnesia_schema:cs2list(Cs2), + TabDef = mnesia_schema:vsn_cs2list(Cs2), Op = {op, change_table_frag, {del_node, Node}, TabDef}, [Op]; false -> diff --git a/lib/mnesia/src/mnesia_monitor.erl b/lib/mnesia/src/mnesia_monitor.erl index e110ad3241..8cb2e92c08 100644 --- a/lib/mnesia/src/mnesia_monitor.erl +++ b/lib/mnesia/src/mnesia_monitor.erl @@ -536,7 +536,11 @@ handle_info({'EXIT', Pid, R}, State) when Pid == State#state.supervisor -> handle_info({'EXIT', Pid, fatal}, State) when node(Pid) == node() -> dbg_out("~p got FATAL ERROR from: ~p~n",[?MODULE, Pid]), - exit(State#state.supervisor, shutdown), + %% This may hang supervisor if a shutdown happens at the same time as an fatal + %% is in progress + %% exit(State#state.supervisor, shutdown), + %% It is better to kill an innocent process + catch exit(whereis(mnesia_locker), kill), {noreply, State}; handle_info(Msg = {'EXIT',Pid,_}, State) -> diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index 05be474aea..179e15197e 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -39,9 +39,10 @@ change_table_load_order/2, change_table_majority/2, change_table_frag/2, - clear_table/1, +%% clear_table/1, %% removed since it is not a schema op anymore create_table/1, cs2list/1, + vsn_cs2list/1, del_snmp/1, del_table_copy/2, del_table_index/2, @@ -65,6 +66,7 @@ merge_schema/0, merge_schema/1, move_table/3, + normalize_cs/2, opt_create_dir/2, prepare_commit/3, purge_dir/2, @@ -626,6 +628,17 @@ do_insert_schema_ops(Store, [Head | Tail]) -> do_insert_schema_ops(_Store, []) -> ok. +api_list2cs(List) when is_list(List) -> + Name = pick(unknown, name, List, must), + Keys = check_keys(Name, List, record_info(fields, cstruct)), + check_duplicates(Name, Keys), + list2cs(List); +api_list2cs(Other) -> + mnesia:abort({badarg, Other}). + +vsn_cs2list(Cs) -> + cs2list(need_old_cstructs(), Cs). + cs2list(Cs) when is_record(Cs, cstruct) -> Tags = record_info(fields, cstruct), rec2list(Tags, Tags, 2, Cs); @@ -648,7 +661,7 @@ cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 17 -> cs2list(false, Cs) -> cs2list(Cs); -cs2list(ver4_4_18, Cs) -> +cs2list(ver4_4_18, Cs) -> %% Or earlier Orig = record_info(fields, cstruct), Tags = [name,type,ram_copies,disc_copies,disc_only_copies, load_order,access_mode,index,snmp,local_content, @@ -671,13 +684,19 @@ rec2list([], _, _Pos, _Rec) -> rec2list(Tags, [_|Orig], Pos, Rec) -> rec2list(Tags, Orig, Pos+1, Rec). -api_list2cs(List) when is_list(List) -> - Name = pick(unknown, name, List, must), - Keys = check_keys(Name, List, record_info(fields, cstruct)), - check_duplicates(Name, Keys), - list2cs(List); -api_list2cs(Other) -> - mnesia:abort({badarg, Other}). +normalize_cs(Cstructs, Node) -> + %% backward-compatibility hack; normalize before returning + case need_old_cstructs([Node]) of + false -> + Cstructs; + Version -> + %% some other format + [convert_cs(Version, Cs) || Cs <- Cstructs] + end. + +convert_cs(Version, Cs) -> + Fields = [Value || {_, Value} <- cs2list(Version, Cs)], + list_to_tuple([cstruct|Fields]). list2cs(List) when is_list(List) -> Name = pick(unknown, name, List, must), @@ -1048,7 +1067,7 @@ unsafe_make_create_table(Cs) -> Nodes = mnesia_lib:intersect(mnesia_lib:cs_to_nodes(Cs), RunningNodes), Store = Ts#tidstore.store, mnesia_locker:wlock_no_exist(Tid, Store, Tab, Nodes), - [{op, create_table, cs2list(Cs)}]. + [{op, create_table, vsn_cs2list(Cs)}]. check_if_exists(Tab) -> TidTs = get_tid_ts_and_lock(schema, write), @@ -1133,7 +1152,7 @@ make_delete_table2(Tab) -> Cs = val({Tab, cstruct}), ensure_active(Cs), ensure_writable(Tab), - {op, delete_table, cs2list(Cs)}. + {op, delete_table, vsn_cs2list(Cs)}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Change fragmentation of a table @@ -1152,10 +1171,6 @@ do_change_table_frag(Tab, _Change) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Clear a table -%% No need for a schema transaction -clear_table(Tab) -> - schema_transaction(fun() -> do_clear_table(Tab) end). - do_clear_table(schema) -> mnesia:abort({bad_type, schema}); do_clear_table(Tab) -> @@ -1166,7 +1181,7 @@ do_clear_table(Tab) -> make_clear_table(Tab) -> Cs = val({Tab, cstruct}), ensure_writable(Tab), - [{op, clear_table, cs2list(Cs)}]. + [{op, clear_table, vsn_cs2list(Cs)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1206,7 +1221,7 @@ make_add_table_copy(Tab, Node, Storage) -> IsRunning == false -> mnesia:abort({not_active, schema, Node}) end, - [{op, add_table_copy, Storage, Node, cs2list(Cs2)}]. + [{op, add_table_copy, Storage, Node, vsn_cs2list(Cs2)}]. del_table_copy(Tab, Node) -> schema_transaction(fun() -> do_del_table_copy(Tab, Node) end). @@ -1235,11 +1250,11 @@ make_del_table_copy(Tab, Node) -> ensure_not_active(Tab, Node), verify_cstruct(Cs2), Ops = remove_node_from_tabs(val({schema, tables}), Node), - [{op, del_table_copy, ram_copies, Node, cs2list(Cs2)} | Ops]; + [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs2)} | Ops]; _ -> ensure_active(Cs), verify_cstruct(Cs2), - [{op, del_table_copy, Storage, Node, cs2list(Cs2)}] + [{op, del_table_copy, Storage, Node, vsn_cs2list(Cs2)}] end. remove_node_from_tabs([], _Node) -> @@ -1253,7 +1268,7 @@ remove_node_from_tabs([Tab|Rest], Node) -> unknown -> case IsFragModified of true -> - [{op, change_table_frag, {del_node, Node}, cs2list(Cs)} | + [{op, change_table_frag, {del_node, Node}, vsn_cs2list(Cs)} | remove_node_from_tabs(Rest, Node)]; false -> remove_node_from_tabs(Rest, Node) @@ -1262,11 +1277,11 @@ remove_node_from_tabs([Tab|Rest], Node) -> Cs2 = new_cs(Cs, Node, Storage, del), case mnesia_lib:cs_to_nodes(Cs2) of [] -> - [{op, delete_table, cs2list(Cs)} | + [{op, delete_table, vsn_cs2list(Cs)} | remove_node_from_tabs(Rest, Node)]; _Ns -> verify_cstruct(Cs2), - [{op, del_table_copy, ram_copies, Node, cs2list(Cs2)}| + [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs2)}| remove_node_from_tabs(Rest, Node)] end end. @@ -1318,9 +1333,9 @@ make_move_table(Tab, FromNode, ToNode) -> Cs2 = new_cs(Cs, ToNode, Storage, add), Cs3 = new_cs(Cs2, FromNode, Storage, del), verify_cstruct(Cs3), - [{op, add_table_copy, Storage, ToNode, cs2list(Cs2)}, + [{op, add_table_copy, Storage, ToNode, vsn_cs2list(Cs2)}, {op, sync_trans}, - {op, del_table_copy, Storage, FromNode, cs2list(Cs3)}]. + {op, del_table_copy, Storage, FromNode, vsn_cs2list(Cs3)}]. %% end of functions to add and delete nodes to tables %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1357,7 +1372,7 @@ make_change_table_copy_type(Tab, Node, ToS) -> Cs3 = new_cs(Cs2, Node, ToS, add), verify_cstruct(Cs3), - [{op, change_table_copy_type, Node, FromS, ToS, cs2list(Cs3)}]. + [{op, change_table_copy_type, Node, FromS, ToS, vsn_cs2list(Cs3)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% change index functions .... @@ -1383,7 +1398,7 @@ make_add_table_index(Tab, Pos) -> Ix2 = lists:sort([Pos | Ix]), Cs2 = Cs#cstruct{index = Ix2}, verify_cstruct(Cs2), - [{op, add_index, Pos, cs2list(Cs2)}]. + [{op, add_index, Pos, vsn_cs2list(Cs2)}]. del_table_index(Tab, Pos) -> schema_transaction(fun() -> do_del_table_index(Tab, Pos) end). @@ -1404,7 +1419,7 @@ make_del_table_index(Tab, Pos) -> verify(true, lists:member(Pos, Ix), {no_exists, Tab, Pos}), Cs2 = Cs#cstruct{index = lists:delete(Pos, Ix)}, verify_cstruct(Cs2), - [{op, del_index, Pos, cs2list(Cs2)}]. + [{op, del_index, Pos, vsn_cs2list(Cs2)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1427,7 +1442,7 @@ make_add_snmp(Tab, Ustruct) -> verify(true, mnesia_snmp_hook:check_ustruct(Ustruct), Error), Cs2 = Cs#cstruct{snmp = Ustruct}, verify_cstruct(Cs2), - [{op, add_snmp, Ustruct, cs2list(Cs2)}]. + [{op, add_snmp, Ustruct, vsn_cs2list(Cs2)}]. del_snmp(Tab) -> schema_transaction(fun() -> do_del_snmp(Tab) end). @@ -1445,7 +1460,7 @@ make_del_snmp(Tab) -> ensure_active(Cs), Cs2 = Cs#cstruct{snmp = []}, verify_cstruct(Cs2), - [{op, del_snmp, cs2list(Cs2)}]. + [{op, del_snmp, vsn_cs2list(Cs2)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% @@ -1477,26 +1492,26 @@ make_transform(Tab, Fun, NewAttrs, NewRecName) -> [] -> Cs2 = Cs#cstruct{attributes = NewAttrs, record_name = NewRecName}, verify_cstruct(Cs2), - [{op, transform, Fun, cs2list(Cs2)}]; + [{op, transform, Fun, vsn_cs2list(Cs2)}]; PosList -> DelIdx = fun(Pos, Ncs) -> Ix = Ncs#cstruct.index, Ncs1 = Ncs#cstruct{index = lists:delete(Pos, Ix)}, - Op = {op, del_index, Pos, cs2list(Ncs1)}, + Op = {op, del_index, Pos, vsn_cs2list(Ncs1)}, {Op, Ncs1} end, AddIdx = fun(Pos, Ncs) -> Ix = Ncs#cstruct.index, Ix2 = lists:sort([Pos | Ix]), Ncs1 = Ncs#cstruct{index = Ix2}, - Op = {op, add_index, Pos, cs2list(Ncs1)}, + Op = {op, add_index, Pos, vsn_cs2list(Ncs1)}, {Op, Ncs1} end, {DelOps, Cs1} = lists:mapfoldl(DelIdx, Cs, PosList), Cs2 = Cs1#cstruct{attributes = NewAttrs, record_name = NewRecName}, {AddOps, Cs3} = lists:mapfoldl(AddIdx, Cs2, PosList), verify_cstruct(Cs3), - lists:flatten([DelOps, {op, transform, Fun, cs2list(Cs2)}, AddOps]) + lists:flatten([DelOps, {op, transform, Fun, vsn_cs2list(Cs2)}, AddOps]) end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1520,7 +1535,7 @@ make_change_table_access_mode(Tab, Mode) -> verify(false, OldMode == Mode, {already_exists, Tab, Mode}), Cs2 = Cs#cstruct{access_mode = Mode}, verify_cstruct(Cs2), - [{op, change_table_access_mode, cs2list(Cs2), OldMode, Mode}]. + [{op, change_table_access_mode, vsn_cs2list(Cs2), OldMode, Mode}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1541,7 +1556,7 @@ make_change_table_load_order(Tab, LoadOrder) -> OldLoadOrder = Cs#cstruct.load_order, Cs2 = Cs#cstruct{load_order = LoadOrder}, verify_cstruct(Cs2), - [{op, change_table_load_order, cs2list(Cs2), OldLoadOrder, LoadOrder}]. + [{op, change_table_load_order, vsn_cs2list(Cs2), OldLoadOrder, LoadOrder}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1571,14 +1586,14 @@ make_change_table_majority(Tab, Majority) -> ensure_active(CsT), CsT2 = CsT#cstruct{majority = Majority}, verify_cstruct(CsT2), - {op, change_table_majority, cs2list(CsT2), + {op, change_table_majority, vsn_cs2list(CsT2), OldMajority, Majority} end, FragNames); false -> []; {_, _} -> mnesia:abort({bad_type, Tab}) end, verify_cstruct(Cs2), - [{op, change_table_majority, cs2list(Cs2), OldMajority, Majority} | FragOps]. + [{op, change_table_majority, vsn_cs2list(Cs2), OldMajority, Majority} | FragOps]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1619,7 +1634,7 @@ make_write_table_properties(Tab, [Prop | Props], Cs) -> MergedProps = lists:merge(DelProps, [Prop]), Cs2 = Cs#cstruct{user_properties = MergedProps}, verify_cstruct(Cs2), - [{op, write_property, cs2list(Cs2), Prop} | + [{op, write_property, vsn_cs2list(Cs2), Prop} | make_write_table_properties(Tab, Props, Cs2)]; make_write_table_properties(_Tab, [], _Cs) -> []. @@ -1740,7 +1755,7 @@ make_delete_table_properties(Tab, [PropKey | PropKeys], Cs) -> Props = lists:keydelete(PropKey, 1, OldProps), Cs2 = Cs#cstruct{user_properties = Props}, verify_cstruct(Cs2), - [{op, delete_property, cs2list(Cs2), PropKey} | + [{op, delete_property, vsn_cs2list(Cs2), PropKey} | make_delete_table_properties(Tab, PropKeys, Cs2)]; make_delete_table_properties(_Tab, [], _Cs) -> []. @@ -2166,12 +2181,17 @@ receive_sync(Nodes, Pids) -> {abort, Else} end. -lock_del_table(Tab, Node, Cs, Father) -> +lock_del_table(Tab, NewNode, Cs0, Father) -> Ns = val({schema, active_replicas}), process_flag(trap_exit,true), Lock = fun() -> mnesia:write_lock_table(Tab), - {Res, []} = rpc:multicall(Ns, ?MODULE, set_where_to_read, [Tab, Node, Cs]), + %% Sigh using cs record + Set = fun(Node) -> + [Cs] = normalize_cs([Cs0], Node), + rpc:call(Node, ?MODULE, set_where_to_read, [Tab, NewNode, Cs]) + end, + Res = [Set(Node) || Node <- Ns], Filter = fun(ok) -> false; ({badrpc, {'EXIT', {undef, _}}}) -> @@ -2353,11 +2373,12 @@ undo_prepare_op(Tid, {op, add_table_copy, Storage, Node, TabDef}) -> undo_prepare_op(_Tid, {op, del_table_copy, _, Node, TabDef}) when Node == node() -> + WriteLocker = get(mnesia_lock), + WriteLocker =/= undefined andalso (WriteLocker ! die), Cs = list2cs(TabDef), Tab = Cs#cstruct.name, mnesia_lib:set({Tab, where_to_read}, Node); - undo_prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}) when N == node() -> Cs = list2cs(TabDef), @@ -2829,6 +2850,9 @@ fetch_cstructs(Node) -> rpc:call(Node, mnesia_controller, get_remote_cstructs, []) end. +need_old_cstructs() -> + need_old_cstructs(val({schema, where_to_write})). + need_old_cstructs(Nodes) -> Filter = fun(Node) -> not mnesia_monitor:needs_protocol_conversion(Node) end, case lists:dropwhile(Filter, Nodes) of diff --git a/lib/mnesia/test/mnesia_evil_backup.erl b/lib/mnesia/test/mnesia_evil_backup.erl index 63f4146d98..9e0a8db1ae 100644 --- a/lib/mnesia/test/mnesia_evil_backup.erl +++ b/lib/mnesia/test/mnesia_evil_backup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -244,9 +244,9 @@ restore(Config, Op) -> [rpc:call(Node, ?MODULE, check_tab, [Res31, ?LINE]) || Node <- Nodes], %% Restore all tables on it's nodes - mnesia_schema:clear_table(Tab1), - mnesia_schema:clear_table(Tab2), - mnesia_schema:clear_table(Tab3), + mnesia:clear_table(Tab1), + mnesia:clear_table(Tab2), + mnesia:clear_table(Tab3), [mnesia:dirty_write({Tab1, N, N+1}) || N <- lists:seq(1, 11)], [mnesia:dirty_write({Tab2, N, N+1}) || N <- lists:seq(1, 11)], [mnesia:dirty_write({Tab3, N, N+1}) || N <- lists:seq(1, 11)], diff --git a/lib/mnesia/test/mnesia_evil_coverage_test.erl b/lib/mnesia/test/mnesia_evil_coverage_test.erl index 668eba176f..17d6c6c212 100644 --- a/lib/mnesia/test/mnesia_evil_coverage_test.erl +++ b/lib/mnesia/test/mnesia_evil_coverage_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1795,7 +1795,7 @@ subscribe_extended(Config) when is_list(Config) -> ?match({mnesia_table_event, {delete, schema, {schema, Tab1}, [{schema, Tab1, _}],_}}, recv_event()), ?match({mnesia_table_event, {write, schema, {schema, Tab1, _}, [], _}}, recv_event()), - ?match({atomic, ok}, mnesia_schema:clear_table(Tab2)), + ?match({atomic, ok}, mnesia:clear_table(Tab2)), ?match({mnesia_table_event, {delete, schema, {schema, Tab2}, [{schema, Tab2, _}],_}}, recv_event()), ?match({mnesia_table_event, {write, schema, {schema, Tab2, _}, [], _}}, recv_event()), diff --git a/lib/mnesia/test/mnesia_install_test.erl b/lib/mnesia/test/mnesia_install_test.erl index 5d55fcac0e..3a2d44aa95 100644 --- a/lib/mnesia/test/mnesia_install_test.erl +++ b/lib/mnesia/test/mnesia_install_test.erl @@ -205,7 +205,7 @@ silly_upgrade(Config) when is_list(Config) -> ?match(ok, mnesia:install_fallback(Bup2)), file:delete(Bup2), %% Will generate intentional crash, fatal error - ?match([], mnesia_test_lib:stop_mnesia([Node2])), + ?match([], mnesia_test_lib:stop_mnesia([Node2])), wait_till_dead([Node1, Node2]), ?match([], mnesia_test_lib:start_mnesia([Node1, Node2], [Tab1, Tab2])), ?match(match, verify_state(Tab1, Tab2, CpState)), @@ -213,22 +213,29 @@ silly_upgrade(Config) when is_list(Config) -> ?match(ok, mnesia:install_fallback(Bup)), file:delete(Bup), %% Will generate intentional crash, fatal error - ?match([], mnesia_test_lib:stop_mnesia([Node1, Node2])), + ?match([], mnesia_test_lib:stop_mnesia([Node1, Node2])), wait_till_dead([Node1, Node2]), ?match([], mnesia_test_lib:start_mnesia([Node1, Node2], [Tab1, Tab2])), CpState2 = [X || X <- CpState, element(1, X) /= Tab1], ?match(match, verify_state(Tab1, Tab2, CpState2)), ?verify_mnesia(Nodes, []). -wait_till_dead([]) -> ok; -wait_till_dead([N|Ns]) -> +wait_till_dead([]) -> + ok; %% timer:sleep(5); +wait_till_dead(Repeat = [N|Ns]) -> Apps = rpc:call(N, application, which_applications, []), case lists:keymember(mnesia, 1, Apps) of - true -> + true -> timer:sleep(10), - wait_till_dead([N|Ns]); - false -> - wait_till_dead(Ns) + wait_till_dead(Repeat); + false -> + case rpc:call(N, erlang, whereis, [mnesia_monitor]) of + undefined -> + wait_till_dead(Ns); + _ -> + timer:sleep(10), + wait_till_dead(Repeat) + end end. add_some_records(Tab1, Tab2, Old) -> diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index a21ab007ef..ebf79dd2ae 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.5 +MNESIA_VSN = 4.5.1 diff --git a/lib/orber/priv/Makefile b/lib/orber/priv/Makefile index 2847727035..2847727035 100755..100644 --- a/lib/orber/priv/Makefile +++ b/lib/orber/priv/Makefile diff --git a/lib/orber/priv/blank.html b/lib/orber/priv/blank.html index 44e86908a0..44e86908a0 100755..100644 --- a/lib/orber/priv/blank.html +++ b/lib/orber/priv/blank.html diff --git a/lib/orber/priv/info_frames.html b/lib/orber/priv/info_frames.html index 75456a67a4..75456a67a4 100755..100644 --- a/lib/orber/priv/info_frames.html +++ b/lib/orber/priv/info_frames.html diff --git a/lib/orber/priv/main_frame.html b/lib/orber/priv/main_frame.html index 056a92812e..056a92812e 100755..100644 --- a/lib/orber/priv/main_frame.html +++ b/lib/orber/priv/main_frame.html diff --git a/lib/orber/priv/orber.tool b/lib/orber/priv/orber.tool index 910a7d7e45..910a7d7e45 100755..100644 --- a/lib/orber/priv/orber.tool +++ b/lib/orber/priv/orber.tool diff --git a/lib/orber/priv/orber_help.txt b/lib/orber/priv/orber_help.txt index a6580dc30a..a6580dc30a 100755..100644 --- a/lib/orber/priv/orber_help.txt +++ b/lib/orber/priv/orber_help.txt diff --git a/lib/orber/priv/start_info.html b/lib/orber/priv/start_info.html index 0ad521c90a..0ad521c90a 100755..100644 --- a/lib/orber/priv/start_info.html +++ b/lib/orber/priv/start_info.html diff --git a/lib/percept/doc/src/part_notes.xml b/lib/percept/doc/src/part_notes.xml index 4965e67640..4965e67640 100755..100644 --- a/lib/percept/doc/src/part_notes.xml +++ b/lib/percept/doc/src/part_notes.xml diff --git a/lib/percept/test/percept_SUITE_data/ipc-dist.dat b/lib/percept/test/percept_SUITE_data/ipc-dist.dat Binary files differindex 14ab6c0c5d..14ab6c0c5d 100755..100644 --- a/lib/percept/test/percept_SUITE_data/ipc-dist.dat +++ b/lib/percept/test/percept_SUITE_data/ipc-dist.dat diff --git a/lib/public_key/asn1/DSS.asn1 b/lib/public_key/asn1/DSS.asn1 index 77aca3808b..77aca3808b 100755..100644 --- a/lib/public_key/asn1/DSS.asn1 +++ b/lib/public_key/asn1/DSS.asn1 diff --git a/lib/public_key/asn1/InformationFramework.asn1 b/lib/public_key/asn1/InformationFramework.asn1 new file mode 100644 index 0000000000..40fbd11a2a --- /dev/null +++ b/lib/public_key/asn1/InformationFramework.asn1 @@ -0,0 +1,682 @@ +InformationFramework {joint-iso-itu-t ds(5) module(1) informationFramework(1) + 6} DEFINITIONS ::= +BEGIN + +-- EXPORTS All +-- The types and values defined in this module are exported for use in the other ASN.1 modules contained +-- within the Directory Specifications, and for the use of other applications which will use them to access +-- Directory services. Other applications may use them for their own purposes, but this will not constrain +-- extensions and modifications needed to maintain or improve the Directory service. +IMPORTS + -- from ITU-T Rec. X.501 | ISO/IEC 9594-2 + directoryAbstractService, id-ar, id-at, id-mr, id-nf, id-oa, id-oc, + id-sc, selectedAttributeTypes, serviceAdministration + FROM UsefulDefinitions {joint-iso-itu-t ds(5) module(1) + usefulDefinitions(0) 6} + SearchRule + FROM ServiceAdministration serviceAdministration + -- from ITU-T Rec. X.511 | ISO/IEC 9594-3 + TypeAndContextAssertion + FROM DirectoryAbstractService directoryAbstractService + -- from ITU-T Rec. X.520 | ISO/IEC 9594-6 + booleanMatch, commonName, generalizedTimeMatch, generalizedTimeOrderingMatch, + integerFirstComponentMatch, integerMatch, integerOrderingMatch, + objectIdentifierFirstComponentMatch, UnboundedDirectoryString + FROM SelectedAttributeTypes selectedAttributeTypes; + +-- attribute data types +Attribute{ATTRIBUTE:SupportedAttributes} ::= SEQUENCE { + type ATTRIBUTE.&id({SupportedAttributes}), + values + SET SIZE (0..MAX) OF ATTRIBUTE.&Type({SupportedAttributes}{@type}), + valuesWithContext + SET SIZE (1..MAX) OF + SEQUENCE {value ATTRIBUTE.&Type({SupportedAttributes}{@type}), + contextList SET SIZE (1..MAX) OF Context} OPTIONAL +} + +AttributeType ::= ATTRIBUTE.&id + +AttributeValue ::= ATTRIBUTE.&Type + +Context ::= SEQUENCE { + contextType CONTEXT.&id({SupportedContexts}), + contextValues + SET SIZE (1..MAX) OF CONTEXT.&Type({SupportedContexts}{@contextType}), + fallback BOOLEAN DEFAULT FALSE +} + +AttributeValueAssertion ::= SEQUENCE { + type ATTRIBUTE.&id({SupportedAttributes}), + assertion + ATTRIBUTE.&equality-match.&AssertionType + ({SupportedAttributes}{@type}), + assertedContexts + CHOICE {allContexts [0] NULL, + selectedContexts [1] SET SIZE (1..MAX) OF ContextAssertion + } OPTIONAL +} + +ContextAssertion ::= SEQUENCE { + contextType CONTEXT.&id({SupportedContexts}), + contextValues + SET SIZE (1..MAX) OF + CONTEXT.&Assertion({SupportedContexts}{@contextType}) +} + +AttributeTypeAssertion ::= SEQUENCE { + type ATTRIBUTE.&id({SupportedAttributes}), + assertedContexts SEQUENCE SIZE (1..MAX) OF ContextAssertion OPTIONAL +} + +-- Definition of the following information object set is deferred, perhaps to standardized +-- profiles or to protocol implementation conformance statements. The set is required to +-- specify a table constraint on the values component of Attribute, the value component +-- of AttributeTypeAndValue, and the assertion component of AttributeValueAssertion. +SupportedAttributes ATTRIBUTE ::= + {objectClass | aliasedEntryName, ...} + +-- Definition of the following information object set is deferred, perhaps to standardized +-- profiles or to protocol implementation conformance statements. The set is required to +-- specify a table constraint on the context specifications +SupportedContexts CONTEXT ::= + {...} + +-- naming data types +Name ::= CHOICE { -- only one possibility for now --rdnSequence RDNSequence +} + +RDNSequence ::= SEQUENCE OF RelativeDistinguishedName + +DistinguishedName ::= RDNSequence + +RelativeDistinguishedName ::= + SET SIZE (1..MAX) OF AttributeTypeAndDistinguishedValue + +AttributeTypeAndDistinguishedValue ::= SEQUENCE { + type ATTRIBUTE.&id({SupportedAttributes}), + value ATTRIBUTE.&Type({SupportedAttributes}{@type}), + primaryDistinguished BOOLEAN DEFAULT TRUE, + valuesWithContext + SET SIZE (1..MAX) OF + SEQUENCE {distingAttrValue + [0] ATTRIBUTE.&Type({SupportedAttributes}{@type}) + OPTIONAL, + contextList SET SIZE (1..MAX) OF Context} OPTIONAL +} + +-- subtree data types +SubtreeSpecification ::= SEQUENCE { + base [0] LocalName DEFAULT {}, + COMPONENTS OF ChopSpecification, + specificationFilter [4] Refinement OPTIONAL +} + +-- empty sequence specifies whole administrative area +LocalName ::= RDNSequence + +ChopSpecification ::= SEQUENCE { + specificExclusions + [1] SET SIZE (1..MAX) OF + CHOICE {chopBefore [0] LocalName, + chopAfter [1] LocalName} OPTIONAL, + minimum [2] BaseDistance DEFAULT 0, + maximum [3] BaseDistance OPTIONAL +} + +BaseDistance ::= INTEGER(0..MAX) + +Refinement ::= CHOICE { + item [0] OBJECT-CLASS.&id, + and [1] SET SIZE (1..MAX) OF Refinement, + or [2] SET SIZE (1..MAX) OF Refinement, + not [3] Refinement +} + +-- OBJECT-CLASS information object class specification +OBJECT-CLASS ::= CLASS { + &Superclasses OBJECT-CLASS OPTIONAL, + &kind ObjectClassKind DEFAULT structural, + &MandatoryAttributes ATTRIBUTE OPTIONAL, + &OptionalAttributes ATTRIBUTE OPTIONAL, + &id OBJECT IDENTIFIER UNIQUE +} +WITH SYNTAX { + [SUBCLASS OF &Superclasses] + [KIND &kind] + [MUST CONTAIN &MandatoryAttributes] + [MAY CONTAIN &OptionalAttributes] + ID &id +} + +ObjectClassKind ::= ENUMERATED {abstract(0), structural(1), auxiliary(2)} + +-- object classes +top OBJECT-CLASS ::= { + KIND abstract + MUST CONTAIN {objectClass} + ID id-oc-top +} + +alias OBJECT-CLASS ::= { + SUBCLASS OF {top} + MUST CONTAIN {aliasedEntryName} + ID id-oc-alias +} + +parent OBJECT-CLASS ::= {KIND abstract + ID id-oc-parent +} + +child OBJECT-CLASS ::= {KIND auxiliary + ID id-oc-child +} + +-- ATTRIBUTE information object class specification +ATTRIBUTE ::= CLASS { + &derivation ATTRIBUTE OPTIONAL, + &Type OPTIONAL, -- either &Type or &derivation required + &equality-match MATCHING-RULE OPTIONAL, + &ordering-match MATCHING-RULE OPTIONAL, + &substrings-match MATCHING-RULE OPTIONAL, + &single-valued BOOLEAN DEFAULT FALSE, + &collective BOOLEAN DEFAULT FALSE, + &dummy BOOLEAN DEFAULT FALSE, + -- operational extensions + &no-user-modification BOOLEAN DEFAULT FALSE, + &usage AttributeUsage DEFAULT userApplications, + &id OBJECT IDENTIFIER UNIQUE +} +WITH SYNTAX { + [SUBTYPE OF &derivation] + [WITH SYNTAX &Type] + [EQUALITY MATCHING RULE &equality-match] + [ORDERING MATCHING RULE &ordering-match] + [SUBSTRINGS MATCHING RULE &substrings-match] + [SINGLE VALUE &single-valued] + [COLLECTIVE &collective] + [DUMMY &dummy] + [NO USER MODIFICATION &no-user-modification] + [USAGE &usage] + ID &id +} + +AttributeUsage ::= ENUMERATED { + userApplications(0), directoryOperation(1), distributedOperation(2), + dSAOperation(3)} + +-- attributes +objectClass ATTRIBUTE ::= { + WITH SYNTAX OBJECT IDENTIFIER + EQUALITY MATCHING RULE objectIdentifierMatch + ID id-at-objectClass +} + +aliasedEntryName ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + SINGLE VALUE TRUE + ID id-at-aliasedEntryName +} + +-- MATCHING-RULE information object class specification +MATCHING-RULE ::= CLASS { + &ParentMatchingRules MATCHING-RULE OPTIONAL, + &AssertionType OPTIONAL, + &uniqueMatchIndicator ATTRIBUTE OPTIONAL, + &id OBJECT IDENTIFIER UNIQUE +} +WITH SYNTAX { + [PARENT &ParentMatchingRules] + [SYNTAX &AssertionType] + [UNIQUE-MATCH-INDICATOR &uniqueMatchIndicator] + ID &id +} + +-- matching rules +objectIdentifierMatch MATCHING-RULE ::= { + SYNTAX OBJECT IDENTIFIER + ID id-mr-objectIdentifierMatch +} + +distinguishedNameMatch MATCHING-RULE ::= { + SYNTAX DistinguishedName + ID id-mr-distinguishedNameMatch +} + +MAPPING-BASED-MATCHING{SelectedBy, BOOLEAN:combinable, MappingResult, + OBJECT IDENTIFIER:matchingRule} ::= CLASS { + &selectBy SelectedBy OPTIONAL, + &ApplicableTo ATTRIBUTE, + &subtypesIncluded BOOLEAN DEFAULT TRUE, + &combinable BOOLEAN(combinable), + &mappingResults MappingResult OPTIONAL, + &userControl BOOLEAN DEFAULT FALSE, + &exclusive BOOLEAN DEFAULT TRUE, + &matching-rule MATCHING-RULE.&id(matchingRule), + &id OBJECT IDENTIFIER UNIQUE +} +WITH SYNTAX { + [SELECT BY &selectBy] + APPLICABLE TO &ApplicableTo + [SUBTYPES INCLUDED &subtypesIncluded] + COMBINABLE &combinable + [MAPPING RESULTS &mappingResults] + [USER CONTROL &userControl] + [EXCLUSIVE &exclusive] + MATCHING RULE &matching-rule + ID &id +} + +-- NAME-FORM information object class specification +NAME-FORM ::= CLASS { + &namedObjectClass OBJECT-CLASS, + &MandatoryAttributes ATTRIBUTE, + &OptionalAttributes ATTRIBUTE OPTIONAL, + &id OBJECT IDENTIFIER UNIQUE +} +WITH SYNTAX { + NAMES &namedObjectClass + WITH ATTRIBUTES &MandatoryAttributes + [AND OPTIONALLY &OptionalAttributes] + ID &id +} + +-- STRUCTURE-RULE class and DIT structure rule data types +DITStructureRule ::= SEQUENCE { + ruleIdentifier RuleIdentifier, + -- shall be unique within the scope of the subschema + nameForm NAME-FORM.&id, + superiorStructureRules SET SIZE (1..MAX) OF RuleIdentifier OPTIONAL +} + +RuleIdentifier ::= INTEGER + +STRUCTURE-RULE ::= CLASS { + &nameForm NAME-FORM, + &SuperiorStructureRules STRUCTURE-RULE OPTIONAL, + &id RuleIdentifier +} +WITH SYNTAX { + NAME FORM &nameForm + [SUPERIOR RULES &SuperiorStructureRules] + ID &id +} + +-- DIT content rule data type and CONTENT-RULE class +DITContentRule ::= SEQUENCE { + structuralObjectClass OBJECT-CLASS.&id, + auxiliaries SET SIZE (1..MAX) OF OBJECT-CLASS.&id OPTIONAL, + mandatory [1] SET SIZE (1..MAX) OF ATTRIBUTE.&id OPTIONAL, + optional [2] SET SIZE (1..MAX) OF ATTRIBUTE.&id OPTIONAL, + precluded [3] SET SIZE (1..MAX) OF ATTRIBUTE.&id OPTIONAL +} + +CONTENT-RULE ::= CLASS { + &structuralClass OBJECT-CLASS.&id UNIQUE, + &Auxiliaries OBJECT-CLASS OPTIONAL, + &Mandatory ATTRIBUTE OPTIONAL, + &Optional ATTRIBUTE OPTIONAL, + &Precluded ATTRIBUTE OPTIONAL +} +WITH SYNTAX { + STRUCTURAL OBJECT-CLASS &structuralClass + [AUXILIARY OBJECT-CLASSES &Auxiliaries] + [MUST CONTAIN &Mandatory] + [MAY CONTAIN &Optional] + [MUST-NOT CONTAIN &Precluded] +} + +CONTEXT ::= CLASS { + &Type , + &DefaultValue OPTIONAL, + &Assertion OPTIONAL, + &absentMatch BOOLEAN DEFAULT TRUE, + &id OBJECT IDENTIFIER UNIQUE +} +WITH SYNTAX { + WITH SYNTAX &Type + [DEFAULT-VALUE &DefaultValue] + [ASSERTED AS &Assertion] + [ABSENT-MATCH &absentMatch] + ID &id +} + +DITContextUse ::= SEQUENCE { + attributeType ATTRIBUTE.&id, + mandatoryContexts [1] SET SIZE (1..MAX) OF CONTEXT.&id OPTIONAL, + optionalContexts [2] SET SIZE (1..MAX) OF CONTEXT.&id OPTIONAL +} + +DIT-CONTEXT-USE-RULE ::= CLASS { + &attributeType ATTRIBUTE.&id UNIQUE, + &Mandatory CONTEXT OPTIONAL, + &Optional CONTEXT OPTIONAL +} +WITH SYNTAX { + ATTRIBUTE TYPE &attributeType + [MANDATORY CONTEXTS &Mandatory] + [OPTIONAL CONTEXTS &Optional] +} + +FRIENDS ::= CLASS { + &anchor ATTRIBUTE.&id UNIQUE, + &Friends ATTRIBUTE +}WITH SYNTAX {ANCHOR &anchor + FRIENDS &Friends +} + +-- system schema information objects +-- object classes +subentry OBJECT-CLASS ::= { + SUBCLASS OF {top} + KIND structural + MUST CONTAIN {commonName | subtreeSpecification} + ID id-sc-subentry +} + +subentryNameForm NAME-FORM ::= { + NAMES subentry + WITH ATTRIBUTES {commonName} + ID id-nf-subentryNameForm +} + +subtreeSpecification ATTRIBUTE ::= { + WITH SYNTAX SubtreeSpecification + USAGE directoryOperation + ID id-oa-subtreeSpecification +} + +administrativeRole ATTRIBUTE ::= { + WITH SYNTAX OBJECT-CLASS.&id + EQUALITY MATCHING RULE objectIdentifierMatch + USAGE directoryOperation + ID id-oa-administrativeRole +} + +createTimestamp ATTRIBUTE ::= { + WITH SYNTAX GeneralizedTime + -- as per 46.3 b) or c) of ITU-T Rec. X.680 | ISO/IEC 8824-1 + EQUALITY MATCHING RULE generalizedTimeMatch + ORDERING MATCHING RULE generalizedTimeOrderingMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-createTimestamp +} + +modifyTimestamp ATTRIBUTE ::= { + WITH SYNTAX GeneralizedTime + -- as per 46.3 b) or c) of ITU-T Rec. X.680 | ISO/IEC 8824-1 + EQUALITY MATCHING RULE generalizedTimeMatch + ORDERING MATCHING RULE generalizedTimeOrderingMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-modifyTimestamp +} + +subschemaTimestamp ATTRIBUTE ::= { + WITH SYNTAX GeneralizedTime + -- as per 46.3 b) or c) of ITU-T Rec. X.680 | ISO/IEC 8824-1 + EQUALITY MATCHING RULE generalizedTimeMatch + ORDERING MATCHING RULE generalizedTimeOrderingMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-subschemaTimestamp +} + +creatorsName ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-creatorsName +} + +modifiersName ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-modifiersName +} + +subschemaSubentryList ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-subschemaSubentryList +} + +accessControlSubentryList ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-accessControlSubentryList +} + +collectiveAttributeSubentryList ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-collectiveAttributeSubentryList +} + +contextDefaultSubentryList ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-contextDefaultSubentryList +} + +serviceAdminSubentryList ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-serviceAdminSubentryList +} + +hasSubordinates ATTRIBUTE ::= { + WITH SYNTAX BOOLEAN + EQUALITY MATCHING RULE booleanMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-hasSubordinates +} + +accessControlSubentry OBJECT-CLASS ::= { + KIND auxiliary + ID id-sc-accessControlSubentry +} + +collectiveAttributeSubentry OBJECT-CLASS ::= { + KIND auxiliary + ID id-sc-collectiveAttributeSubentry +} + +collectiveExclusions ATTRIBUTE ::= { + WITH SYNTAX OBJECT IDENTIFIER + EQUALITY MATCHING RULE objectIdentifierMatch + USAGE directoryOperation + ID id-oa-collectiveExclusions +} + +contextAssertionSubentry OBJECT-CLASS ::= { + KIND auxiliary + MUST CONTAIN {contextAssertionDefaults} + ID id-sc-contextAssertionSubentry +} + +contextAssertionDefaults ATTRIBUTE ::= { + WITH SYNTAX TypeAndContextAssertion + EQUALITY MATCHING RULE objectIdentifierFirstComponentMatch + USAGE directoryOperation + ID id-oa-contextAssertionDefault +} + +serviceAdminSubentry OBJECT-CLASS ::= { + KIND auxiliary + MUST CONTAIN {searchRules} + ID id-sc-serviceAdminSubentry +} + +searchRules ATTRIBUTE ::= { + WITH SYNTAX SearchRuleDescription + EQUALITY MATCHING RULE integerFirstComponentMatch + USAGE directoryOperation + ID id-oa-searchRules +} + +SearchRuleDescription ::= SEQUENCE { + COMPONENTS OF SearchRule, + name [28] SET SIZE (1..MAX) OF UnboundedDirectoryString OPTIONAL, + description [29] UnboundedDirectoryString OPTIONAL +} + +hierarchyLevel ATTRIBUTE ::= { + WITH SYNTAX HierarchyLevel + EQUALITY MATCHING RULE integerMatch + ORDERING MATCHING RULE integerOrderingMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-hierarchyLevel +} + +HierarchyLevel ::= INTEGER + +hierarchyBelow ATTRIBUTE ::= { + WITH SYNTAX HierarchyBelow + EQUALITY MATCHING RULE booleanMatch + SINGLE VALUE TRUE + NO USER MODIFICATION TRUE + USAGE directoryOperation + ID id-oa-hierarchyBelow +} + +HierarchyBelow ::= BOOLEAN + +hierarchyParent ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + SINGLE VALUE TRUE + USAGE directoryOperation + ID id-oa-hierarchyParent +} + +hierarchyTop ATTRIBUTE ::= { + WITH SYNTAX DistinguishedName + EQUALITY MATCHING RULE distinguishedNameMatch + SINGLE VALUE TRUE + USAGE directoryOperation + ID id-oa-hierarchyTop +} + +-- object identifier assignments +-- object classes +id-oc-top OBJECT IDENTIFIER ::= + {id-oc 0} + +id-oc-alias OBJECT IDENTIFIER ::= {id-oc 1} + +id-oc-parent OBJECT IDENTIFIER ::= {id-oc 28} + +id-oc-child OBJECT IDENTIFIER ::= {id-oc 29} + +-- attributes +id-at-objectClass OBJECT IDENTIFIER ::= {id-at 0} + +id-at-aliasedEntryName OBJECT IDENTIFIER ::= {id-at 1} + +-- matching rules +id-mr-objectIdentifierMatch OBJECT IDENTIFIER ::= {id-mr 0} + +id-mr-distinguishedNameMatch OBJECT IDENTIFIER ::= {id-mr 1} + +-- operational attributes +id-oa-excludeAllCollectiveAttributes OBJECT IDENTIFIER ::= + {id-oa 0} + +id-oa-createTimestamp OBJECT IDENTIFIER ::= {id-oa 1} + +id-oa-modifyTimestamp OBJECT IDENTIFIER ::= {id-oa 2} + +id-oa-creatorsName OBJECT IDENTIFIER ::= {id-oa 3} + +id-oa-modifiersName OBJECT IDENTIFIER ::= {id-oa 4} + +id-oa-administrativeRole OBJECT IDENTIFIER ::= {id-oa 5} + +id-oa-subtreeSpecification OBJECT IDENTIFIER ::= {id-oa 6} + +id-oa-collectiveExclusions OBJECT IDENTIFIER ::= {id-oa 7} + +id-oa-subschemaTimestamp OBJECT IDENTIFIER ::= {id-oa 8} + +id-oa-hasSubordinates OBJECT IDENTIFIER ::= {id-oa 9} + +id-oa-subschemaSubentryList OBJECT IDENTIFIER ::= {id-oa 10} + +id-oa-accessControlSubentryList OBJECT IDENTIFIER ::= {id-oa 11} + +id-oa-collectiveAttributeSubentryList OBJECT IDENTIFIER ::= {id-oa 12} + +id-oa-contextDefaultSubentryList OBJECT IDENTIFIER ::= {id-oa 13} + +id-oa-contextAssertionDefault OBJECT IDENTIFIER ::= {id-oa 14} + +id-oa-serviceAdminSubentryList OBJECT IDENTIFIER ::= {id-oa 15} + +id-oa-searchRules OBJECT IDENTIFIER ::= {id-oa 16} + +id-oa-hierarchyLevel OBJECT IDENTIFIER ::= {id-oa 17} + +id-oa-hierarchyBelow OBJECT IDENTIFIER ::= {id-oa 18} + +id-oa-hierarchyParent OBJECT IDENTIFIER ::= {id-oa 19} + +id-oa-hierarchyTop OBJECT IDENTIFIER ::= {id-oa 20} + +-- subentry classes +id-sc-subentry OBJECT IDENTIFIER ::= {id-sc 0} + +id-sc-accessControlSubentry OBJECT IDENTIFIER ::= {id-sc 1} + +id-sc-collectiveAttributeSubentry OBJECT IDENTIFIER ::= {id-sc 2} + +id-sc-contextAssertionSubentry OBJECT IDENTIFIER ::= {id-sc 3} + +id-sc-serviceAdminSubentry OBJECT IDENTIFIER ::= {id-sc 4} + +-- Name forms +id-nf-subentryNameForm OBJECT IDENTIFIER ::= {id-nf 16} + +-- administrative roles +id-ar-autonomousArea OBJECT IDENTIFIER ::= {id-ar 1} + +id-ar-accessControlSpecificArea OBJECT IDENTIFIER ::= {id-ar 2} + +id-ar-accessControlInnerArea OBJECT IDENTIFIER ::= {id-ar 3} + +id-ar-subschemaAdminSpecificArea OBJECT IDENTIFIER ::= {id-ar 4} + +id-ar-collectiveAttributeSpecificArea OBJECT IDENTIFIER ::= {id-ar 5} + +id-ar-collectiveAttributeInnerArea OBJECT IDENTIFIER ::= {id-ar 6} + +id-ar-contextDefaultSpecificArea OBJECT IDENTIFIER ::= {id-ar 7} + +id-ar-serviceSpecificArea OBJECT IDENTIFIER ::= {id-ar 8} + +END -- InformationFramework diff --git a/lib/public_key/asn1/Makefile b/lib/public_key/asn1/Makefile index c4f8d65aa7..2ce1168349 100644 --- a/lib/public_key/asn1/Makefile +++ b/lib/public_key/asn1/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2010. All Rights Reserved. +# Copyright Ericsson AB 2008-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -38,12 +38,12 @@ RELSYSDIR = $(RELEASE_PATH)/lib/public_key-$(VSN) .SUFFIXES: .asn1 .PRECIOUS: %.erl -ASN_TOP = OTP-PUB-KEY +ASN_TOP = OTP-PUB-KEY PKCS-FRAME ASN_MODULES = PKIX1Explicit88 PKIX1Implicit88 PKIX1Algorithms88 \ - PKIXAttributeCertificate PKCS-1 PKCS-3 OTP-PKIX + PKIXAttributeCertificate PKCS-1 PKCS-3 PKCS-8 InformationFramework PKCS5v2-0 OTP-PKIX ASN_ASNS = $(ASN_MODULES:%=%.asn1) -ASN_ERLS = $(ASN_TOP).erl -ASN_HRLS = $(ASN_TOP).hrl +ASN_ERLS = $(ASN_TOP:%=%.erl) +ASN_HRLS = $(ASN_TOP:%=%.hrl) ASN_CONFIGS = OTP-PUB-KEY.asn1config ASN_DBS = $(ASN_MODULES:%=%.asn1db) OTP-PUB-KEY.asn1db ASN_TABLES = $(ASN_MODULES:%=%.table) @@ -82,8 +82,8 @@ docs: %.erl %.hrl: %.set.asn erlc $(ASN_FLAGS) $< -$(HRL_FILES): $(ASN_HRLS) - cp -p $(ASN_HRLS) $(INCLUDE) +$(INCLUDE)/%.hrl: %.hrl + cp -p $< $@ # ---------------------------------------------------- # Release Target @@ -113,3 +113,9 @@ OTP-PUB-KEY.asn1db: PKIX1Algorithms88.asn1 \ PKCS-1.asn1\ PKCS-3.asn1\ OTP-PKIX.asn1 + +$(EBIN)/PKCS-FRAME.beam: PKCS-FRAME.erl PKCS-FRAME.hrl +PKCS-FRAME.erl PKCS-FRAME.hrl: PKCS-FRAME.asn1db +PKCS-FRAME.asn1db: PKCS-8.asn1\ + InformationFramework.asn1\ + PKCS5v2-0.asn1
\ No newline at end of file diff --git a/lib/public_key/asn1/PKCS-1.asn1 b/lib/public_key/asn1/PKCS-1.asn1 index b06f5efa9d..b06f5efa9d 100755..100644 --- a/lib/public_key/asn1/PKCS-1.asn1 +++ b/lib/public_key/asn1/PKCS-1.asn1 diff --git a/lib/public_key/asn1/PKCS-8.asn1 b/lib/public_key/asn1/PKCS-8.asn1 new file mode 100644 index 0000000000..7413519b57 --- /dev/null +++ b/lib/public_key/asn1/PKCS-8.asn1 @@ -0,0 +1,83 @@ +PKCS-8 {iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) pkcs-8(8) + modules(1) pkcs-8(1)} + +-- $Revision: 1.5 $ + +-- This module has been checked for conformance with the ASN.1 +-- standard by the OSS ASN.1 Tools + +DEFINITIONS IMPLICIT TAGS ::= + +BEGIN + +-- EXPORTS All -- +-- All types and values defined in this module is exported for use in other +-- ASN.1 modules. + +IMPORTS + +-- informationFramework +-- FROM UsefulDefinitions {joint-iso-itu-t(2) ds(5) module(1) +-- usefulDefinitions(0) 3} + +Attribute +-- FROM InformationFramework informationFramework + FROM InformationFramework; + +-- This import is really unnecessary since ALGORITHM-IDENTIFIER is defined as a +-- TYPE-IDENTIFIER +-- Renome this import and replace all occurences of ALGORITHM-IDENTIFIER with +-- TYPE-IDENTIFIER as a workaround for weaknesses in the ASN.1 compiler +--AlgorithmIdentifier, ALGORITHM-IDENTIFIER +-- FROM PKCS5v2-0 {iso(1) member-body(2) us(840) rsadsi(113549) +-- pkcs(1) pkcs-5(5) modules(16) pkcs-5(1)}; + +-- Inlined from PKCS5v2-0 since it is the only thing imported from that module +-- AlgorithmIdentifier { ALGORITHM-IDENTIFIER:InfoObjectSet } ::= +AlgorithmIdentifier { TYPE-IDENTIFIER:InfoObjectSet } ::= +SEQUENCE { +-- algorithm ALGORITHM-IDENTIFIER.&id({InfoObjectSet}), + algorithm TYPE-IDENTIFIER.&id({InfoObjectSet}), +-- parameters ALGORITHM-IDENTIFIER.&Type({InfoObjectSet} + parameters TYPE-IDENTIFIER.&Type({InfoObjectSet} + {@algorithm}) OPTIONAL } + +-- Private-key information syntax + +PrivateKeyInfo ::= SEQUENCE { + version Version, +-- privateKeyAlgorithm AlgorithmIdentifier {{PrivateKeyAlgorithms}}, + privateKeyAlgorithm AlgorithmIdentifier {{...}}, + privateKey PrivateKey, + attributes [0] Attributes OPTIONAL } + +Version ::= INTEGER {v1(0)} (v1,...) + +PrivateKey ::= OCTET STRING + +-- Attributes ::= SET OF Attribute +Attributes ::= SET OF Attribute {{...}} + +-- Encrypted private-key information syntax + +EncryptedPrivateKeyInfo ::= SEQUENCE { +-- encryptionAlgorithm AlgorithmIdentifier {{KeyEncryptionAlgorithms}}, + encryptionAlgorithm AlgorithmIdentifier {{...}}, + encryptedData EncryptedData +} + +EncryptedData ::= OCTET STRING + +-- PrivateKeyAlgorithms ALGORITHM-IDENTIFIER ::= { +PrivateKeyAlgorithms TYPE-IDENTIFIER ::= { + ... -- For local profiles +} + +-- KeyEncryptionAlgorithms ALGORITHM-IDENTIFIER ::= { +KeyEncryptionAlgorithms TYPE-IDENTIFIER ::= { + ... -- For local profiles +} + +END + + diff --git a/lib/public_key/asn1/PKCS-FRAME.set.asn b/lib/public_key/asn1/PKCS-FRAME.set.asn new file mode 100644 index 0000000000..a0777ff260 --- /dev/null +++ b/lib/public_key/asn1/PKCS-FRAME.set.asn @@ -0,0 +1,3 @@ +PKCS-8.asn1 +InformationFramework.asn1 +PKCS5v2-0.asn1 diff --git a/lib/public_key/asn1/PKCS5v2-0.asn1 b/lib/public_key/asn1/PKCS5v2-0.asn1 new file mode 100644 index 0000000000..fe7e16c7fa --- /dev/null +++ b/lib/public_key/asn1/PKCS5v2-0.asn1 @@ -0,0 +1,142 @@ +-- PKCS #5 v2.0 ASN.1 Module +-- Revised March 25, 1999 + +-- This module has been checked for conformance with the +-- ASN.1 standard by the OSS ASN.1 Tools + +PKCS5v2-0 {iso(1) member-body(2) us(840) rsadsi(113549) + pkcs(1) pkcs-5(5) modules(16) pkcs5v2-0(1)} + +DEFINITIONS ::= BEGIN + +-- Basic object identifiers + +rsadsi OBJECT IDENTIFIER ::= + {iso(1) member-body(2) us(840) 113549} +pkcs OBJECT IDENTIFIER ::= {rsadsi 1} +pkcs-5 OBJECT IDENTIFIER ::= {pkcs 5} + +-- Basic types and classes + +AlgorithmIdentifier { TYPE-IDENTIFIER:InfoObjectSet } ::= +SEQUENCE { + algorithm TYPE-IDENTIFIER.&id({InfoObjectSet}), + parameters TYPE-IDENTIFIER.&Type({InfoObjectSet} + {@algorithm}) OPTIONAL } + +--ALGORITHM-IDENTIFIER ::= TYPE-IDENTIFIER + +-- PBKDF2 + +-- PBKDF2Algorithms ALGORITHM-IDENTIFIER ::= +-- { {PBKDF2-params IDENTIFIED BY id-PBKDF2}, ...} + +id-PBKDF2 OBJECT IDENTIFIER ::= {pkcs-5 12} + +-- algid-hmacWithSHA1 AlgorithmIdentifier {{PBKDF2-PRFs}} ::= +-- {algorithm id-hmacWithSHA1, parameters NULL : NULL} + +PBKDF2-params ::= SEQUENCE { + salt CHOICE { + specified OCTET STRING, + otherSource AlgorithmIdentifier {{PBKDF2-SaltSources}} + }, + iterationCount INTEGER (1..MAX), + keyLength INTEGER (1..MAX) OPTIONAL, + prf AlgorithmIdentifier {{PBKDF2-PRFs}} DEFAULT +{algorithm id-hmacWithSHA1, parameters NULL : NULL}} +-- algid-hmacWithSHA1 } + +PBKDF2-SaltSources TYPE-IDENTIFIER ::= { ... } + +PBKDF2-PRFs TYPE-IDENTIFIER ::= + { {NULL IDENTIFIED BY id-hmacWithSHA1}, ... } + + -- PBES1 + +PBES1Algorithms TYPE-IDENTIFIER ::= + { {PBEParameter IDENTIFIED BY pbeWithMD2AndDES-CBC} | + {PBEParameter IDENTIFIED BY pbeWithMD2AndRC2-CBC} | + {PBEParameter IDENTIFIED BY pbeWithMD5AndDES-CBC} | + {PBEParameter IDENTIFIED BY pbeWithMD5AndRC2-CBC} | + {PBEParameter IDENTIFIED BY pbeWithSHA1AndDES-CBC} | + {PBEParameter IDENTIFIED BY pbeWithSHA1AndRC2-CBC}, ...} + +pbeWithMD2AndDES-CBC OBJECT IDENTIFIER ::= {pkcs-5 1} +pbeWithMD2AndRC2-CBC OBJECT IDENTIFIER ::= {pkcs-5 4} +pbeWithMD5AndDES-CBC OBJECT IDENTIFIER ::= {pkcs-5 3} +pbeWithMD5AndRC2-CBC OBJECT IDENTIFIER ::= {pkcs-5 6} +pbeWithSHA1AndDES-CBC OBJECT IDENTIFIER ::= {pkcs-5 10} +pbeWithSHA1AndRC2-CBC OBJECT IDENTIFIER ::= {pkcs-5 11} + +PBEParameter ::= SEQUENCE { + salt OCTET STRING (SIZE(8)), + iterationCount INTEGER } + +-- PBES2 + +PBES2Algorithms TYPE-IDENTIFIER ::= + { {PBES2-params IDENTIFIED BY id-PBES2}, ...} + +id-PBES2 OBJECT IDENTIFIER ::= {pkcs-5 13} + +PBES2-params ::= SEQUENCE { + keyDerivationFunc AlgorithmIdentifier {{PBES2-KDFs}}, + encryptionScheme AlgorithmIdentifier {{PBES2-Encs}} } + +PBES2-KDFs TYPE-IDENTIFIER ::= + { {PBKDF2-params IDENTIFIED BY id-PBKDF2}, ... } + +PBES2-Encs TYPE-IDENTIFIER ::= { ... } + +-- PBMAC1 + +PBMAC1Algorithms TYPE-IDENTIFIER ::= + { {PBMAC1-params IDENTIFIED BY id-PBMAC1}, ...} + +id-PBMAC1 OBJECT IDENTIFIER ::= {pkcs-5 14} + +PBMAC1-params ::= SEQUENCE { + keyDerivationFunc AlgorithmIdentifier {{PBMAC1-KDFs}}, + messageAuthScheme AlgorithmIdentifier {{PBMAC1-MACs}} } + +PBMAC1-KDFs TYPE-IDENTIFIER ::= + { {PBKDF2-params IDENTIFIED BY id-PBKDF2}, ... } + +PBMAC1-MACs TYPE-IDENTIFIER ::= { ... } + +-- Supporting techniques + +digestAlgorithm OBJECT IDENTIFIER ::= {rsadsi 2} +encryptionAlgorithm OBJECT IDENTIFIER ::= {rsadsi 3} + +SupportingAlgorithms TYPE-IDENTIFIER ::= + { {NULL IDENTIFIED BY id-hmacWithSHA1} | + {OCTET STRING (SIZE(8)) IDENTIFIED BY desCBC} | + {OCTET STRING (SIZE(8)) IDENTIFIED BY des-EDE3-CBC} | + {RC2-CBC-Parameter IDENTIFIED BY rc2CBC} | + {RC5-CBC-Parameters IDENTIFIED BY rc5-CBC-PAD}, ... } + +id-hmacWithSHA1 OBJECT IDENTIFIER ::= {digestAlgorithm 7} + +desCBC OBJECT IDENTIFIER ::= + {iso(1) identified-organization(3) oiw(14) secsig(3) + algorithms(2) 7} -- from OIW + +des-EDE3-CBC OBJECT IDENTIFIER ::= {encryptionAlgorithm 7} + +rc2CBC OBJECT IDENTIFIER ::= {encryptionAlgorithm 2} + +RC2-CBC-Parameter ::= SEQUENCE { + rc2ParameterVersion INTEGER OPTIONAL, + iv OCTET STRING (SIZE(8)) } + +rc5-CBC-PAD OBJECT IDENTIFIER ::= {encryptionAlgorithm 9} + +RC5-CBC-Parameters ::= SEQUENCE { + version INTEGER {v1-0(16)}, -- (v1-0), + rounds INTEGER (8..127), + blockSizeInBits INTEGER (64 | 128), + iv OCTET STRING OPTIONAL } + +END diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index 9a3832c68b..821e7a2300 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -61,11 +61,14 @@ <p><code>string = [bytes()]</code></p> <p><code>pki_asn1_type() = 'Certificate' | 'RSAPrivateKey'| 'RSAPublicKey' - 'DSAPrivateKey' | 'DSAPublicKey' | 'DHParameter' | 'SubjectPublicKeyInfo'</code></p> + 'DSAPrivateKey' | 'DSAPublicKey' | 'DHParameter' | 'SubjectPublicKeyInfo'| 'PrivateKeyInfo'</code></p> <p><code>pem_entry () = {pki_asn1_type(), binary(), %% DER or encrypted DER - not_encrypted | {"DES-CBC" | "DES-EDE3-CBC", crypto:rand_bytes(8)}}.</code></p> - + not_encrypted | cipher_info()} </code></p> + + <p><code>cipher_info() = {"RC2-CBC | "DES-CBC" | "DES-EDE3-CBC", crypto:rand_bytes(8)} | + 'PBES2-params'} </code></p> + <p><code>rsa_public_key() = #'RSAPublicKey'{}</code></p> <p><code>rsa_private_key() = #'RSAPrivateKey'{} </code></p> @@ -118,7 +121,8 @@ <funcs> <func> - <name>decrypt_private(CipherText, Key [, Options]) -> binary()</name> + <name>decrypt_private(CipherText, Key) -> binary()</name> + <name>decrypt_private(CipherText, Key, Options) -> binary()</name> <fsummary>Public key decryption.</fsummary> <type> <v>CipherText = binary()</v> @@ -131,7 +135,8 @@ </func> <func> - <name>decrypt_public(CipherText, Key [, Options]) - > binary()</name> + <name>decrypt_public(CipherText, Key) - > binary()</name> + <name>decrypt_public(CipherText, Key, Options) - > binary()</name> <fsummary></fsummary> <type> <v>CipherText = binary()</v> @@ -198,7 +203,8 @@ </func> <func> - <name>pem_entry_decode(PemEntry [, Password]) -> term()</name> + <name>pem_entry_decode(PemEntry) -> term()</name> + <name>pem_entry_decode(PemEntry, Password) -> term()</name> <fsummary>Decodes a pem entry.</fsummary> <type> <v> PemEntry = pem_entry() </v> @@ -213,7 +219,8 @@ </func> <func> - <name>pem_entry_encode(Asn1Type, Entity [,{CipherInfo, Password}]) -> pem_entry()</name> + <name>pem_entry_encode(Asn1Type, Entity) -> pem_entry()</name> + <name>pem_entry_encode(Asn1Type, Entity, {CipherInfo, Password}) -> pem_entry()</name> <fsummary> Creates a pem entry that can be fed to pem_encode/1.</fsummary> <type> <v>Asn1Type = pki_asn1_type()</v> @@ -224,7 +231,7 @@ dsa_public_key() and this function will create the appropriate 'SubjectPublicKeyInfo' entry. </d> - <v>CipherInfo = {"DES-CBC" | "DES-EDE3-CBC", crypto:rand_bytes(8)}</v> + <v>CipherInfo = cipher_info()</v> <v>Password = string()</v> </type> <desc> diff --git a/lib/public_key/include/public_key.hrl b/lib/public_key/include/public_key.hrl index 5f97d80f7e..2475295974 100644 --- a/lib/public_key/include/public_key.hrl +++ b/lib/public_key/include/public_key.hrl @@ -23,6 +23,7 @@ -define(public_key, true). -include("OTP-PUB-KEY.hrl"). +-include("PKCS-FRAME.hrl"). -record('SubjectPublicKeyInfoAlgorithm', { algorithm, diff --git a/lib/public_key/src/Makefile b/lib/public_key/src/Makefile index 5a24b02d2a..062c495a65 100644 --- a/lib/public_key/src/Makefile +++ b/lib/public_key/src/Makefile @@ -42,10 +42,11 @@ MODULES = \ public_key \ pubkey_pem \ pubkey_ssh \ + pubkey_pbe \ pubkey_cert \ - pubkey_cert_records + pubkey_cert_records -HRL_FILES = $(INCLUDE)/public_key.hrl +HRL_FILES = $(INCLUDE)/public_key.hrl INTERNAL_HRL_FILES = @@ -109,4 +110,3 @@ release_spec: opt $(INSTALL_DATA) $(TARGET_FILES) $(APP_TARGET) $(APPUP_TARGET) $(RELSYSDIR)/ebin release_docs_spec: - diff --git a/lib/public_key/src/pubkey_pbe.erl b/lib/public_key/src/pubkey_pbe.erl new file mode 100644 index 0000000000..43f6c42f10 --- /dev/null +++ b/lib/public_key/src/pubkey_pbe.erl @@ -0,0 +1,213 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2011-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% Description: Implements Password Based Encryption PKCS-5, RFC-2898 + +-module(pubkey_pbe). + +-include("public_key.hrl"). + +-export([encode/4, decode/4, decrypt_parameters/1]). +-export([pbdkdf1/4, pbdkdf2/6]). + +-define(DEFAULT_SHA_MAC_KEYLEN, 20). +-define(ASN1_OCTET_STR_TAG, 4). +-define(IV_LEN, 8). + +%%==================================================================== +%% Internal application API +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec encode(binary(), string(), string(), term()) -> binary(). +%% +%% Description: Performs password based encoding +%%-------------------------------------------------------------------- +encode(Data, Password, "DES-CBC" = Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:des_cbc_encrypt(Key, IV, Data); + +encode(Data, Password, "DES-EDE3-CBC" = Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, + crypto:des_ede3_cbc_encrypt(Key1, Key2, Key3, IV, Data); + +encode(Data, Password, "RC2-CBC" = Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:rc2_cbc_encrypt(Key, IV, Data). +%%-------------------------------------------------------------------- +-spec decode(binary(), string(), string(), term()) -> binary(). +%% +%% Description: Performs password based decoding +%%-------------------------------------------------------------------- +decode(Data, Password,"DES-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:des_cbc_decrypt(Key, IV, Data); + +decode(Data, Password,"DES-EDE3-CBC" = Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, + crypto:des_ede3_cbc_decrypt(Key1, Key2, Key3, IV, Data); + +decode(Data, Password,"RC2-CBC"= Cipher, KeyDevParams) -> + {Key, IV} = password_to_key_and_iv(Password, Cipher, KeyDevParams), + crypto:rc2_cbc_decrypt(Key, IV, Data). + +%%-------------------------------------------------------------------- +-spec pbdkdf1(string(), iodata(), integer(), atom()) -> binary(). +%% +%% Description: Implements password based decryption key derive function 1. +%% Exported mainly for testing purposes. +%%-------------------------------------------------------------------- +pbdkdf1(_, _, 0, Acc) -> + Acc; +pbdkdf1(Password, Salt, Count, Hash) -> + Result = crypto:Hash([Password, Salt]), + do_pbdkdf1(Result, Count-1, Result, Hash). + +%%-------------------------------------------------------------------- +-spec pbdkdf2(string(), iodata(), integer(), integer(), fun(), integer()) + -> binary(). +%% +%% Description: Implements password based decryption key derive function 2. +%% Exported mainly for testing purposes. +%%-------------------------------------------------------------------- +pbdkdf2(Password, Salt, Count, DerivedKeyLen, Prf, PrfOutputLen)-> + NumBlocks = ceiling(DerivedKeyLen / PrfOutputLen), + NumLastBlockOctets = DerivedKeyLen - (NumBlocks - 1) * PrfOutputLen , + blocks(NumBlocks, NumLastBlockOctets, 1, Password, Salt, + Count, Prf, PrfOutputLen, <<>>). +%%-------------------------------------------------------------------- +-spec decrypt_parameters(#'EncryptedPrivateKeyInfo_encryptionAlgorithm'{}) -> + {Cipher::string(), #'PBES2-params'{}}. +%% +%% Description: Performs ANS1-decoding of encryption parameters. +%%-------------------------------------------------------------------- +decrypt_parameters(#'EncryptedPrivateKeyInfo_encryptionAlgorithm'{ + algorithm = Oid, parameters = Param}) -> + decrypt_parameters(Oid, Param). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +password_to_key_and_iv(Password, _, #'PBES2-params'{} = Params) -> + {Salt, ItrCount, KeyLen, PseudoRandomFunction, PseudoOtputLen, IV} = + key_derivation_params(Params), + <<Key:KeyLen/binary, _/binary>> = + pbdkdf2(Password, Salt, ItrCount, KeyLen, PseudoRandomFunction, PseudoOtputLen), + {Key, IV}; +password_to_key_and_iv(Password, Cipher, Salt) -> + KeyLen = derived_key_length(Cipher, undefined), + <<Key:KeyLen/binary, _/binary>> = + pem_encrypt(<<>>, Password, Salt, ceiling(KeyLen div 16), <<>>, md5), + %% Old PEM encryption does not use standard encryption method + %% pbdkdf1 and uses then salt as IV + {Key, Salt}. + +pem_encrypt(_, _, _, 0, Acc, _) -> + Acc; +pem_encrypt(Prev, Password, Salt, Count, Acc, Hash) -> + Result = crypto:Hash([Prev, Password, Salt]), + pem_encrypt(Result, Password, Salt, Count-1 , <<Acc/binary, Result/binary>>, Hash). + +do_pbdkdf1(_, 0, Acc, _) -> + Acc; +do_pbdkdf1(Prev, Count, Acc, Hash) -> + Result = crypto:Hash(Prev), + do_pbdkdf1(Result, Count-1 , <<Result/binary, Acc/binary>>, Hash). + +iv(#'PBES2-params_encryptionScheme'{algorithm = Algo, + parameters = ASNIV}) when (Algo == ?'desCBC') or + (Algo == ?'des-EDE3-CBC') -> + %% This is an so called open ASN1-type that in this + %% case will be an octet-string of length 8 + <<?ASN1_OCTET_STR_TAG, ?IV_LEN, IV:?IV_LEN/binary>> = ASNIV, + IV; +iv(#'PBES2-params_encryptionScheme'{algorithm = ?'rc2CBC', + parameters = ASN1IV}) -> + {ok, #'RC2-CBC-Parameter'{iv = IV}} = 'PKCS-FRAME':decode('RC2-CBC-Parameter', ASN1IV), + iolist_to_binary(IV). + +blocks(1, N, Index, Password, Salt, Count, Prf, PrfLen, Acc) -> + <<XorSum:N/binary, _/binary>> = xor_sum(Password, Salt, Count, Index, Prf, PrfLen), + <<Acc/binary, XorSum/binary>>; +blocks(NumBlocks, N, Index, Password, Salt, Count, Prf, PrfLen, Acc) -> + XorSum = xor_sum(Password, Salt, Count, Index, Prf, PrfLen), + blocks(NumBlocks -1, N, Index +1, Password, Salt, Count, Prf, + PrfLen, <<Acc/binary, XorSum/binary>>). + +xor_sum(Password, Salt, Count, Index, Prf, PrfLen) -> + Result = Prf(Password, [Salt,<<Index:32/unsigned-big-integer>>], PrfLen), + do_xor_sum(Prf, PrfLen, Result, Password, Count-1, Result). + +do_xor_sum(_, _, _, _, 0, Acc) -> + Acc; +do_xor_sum(Prf, PrfLen, Prev, Password, Count, Acc)-> + Result = Prf(Password, Prev, PrfLen), + do_xor_sum(Prf, PrfLen, Result, Password, Count-1, crypto:exor(Acc, Result)). + +decrypt_parameters(?'id-PBES2', DekParams) -> + {ok, Params} = 'PKCS-FRAME':decode('PBES2-params', DekParams), + {cipher(Params#'PBES2-params'.encryptionScheme), Params}. + +key_derivation_params(#'PBES2-params'{keyDerivationFunc = KeyDerivationFunc, + encryptionScheme = EncScheme}) -> + #'PBES2-params_keyDerivationFunc'{algorithm = ?'id-PBKDF2', + parameters = + #'PBKDF2-params'{salt = {specified, OctetSalt}, + iterationCount = Count, + keyLength = Length, + prf = Prf}} = KeyDerivationFunc, + #'PBES2-params_encryptionScheme'{algorithm = Algo} = EncScheme, + {PseudoRandomFunction, PseudoOtputLen} = pseudo_random_function(Prf), + KeyLen = derived_key_length(Algo, Length), + {OctetSalt, Count, KeyLen, + PseudoRandomFunction, PseudoOtputLen, iv(EncScheme)}. + +%% This function currently matches a tuple that ougth to be the value +%% ?'id-hmacWithSHA1, but we need some kind of ASN1-fix for this. +pseudo_random_function(#'PBKDF2-params_prf'{algorithm = + {_,_, _,'id-hmacWithSHA1'}}) -> + {fun crypto:sha_mac/3, pseudo_output_length(?'id-hmacWithSHA1')}; +pseudo_random_function(#'PBKDF2-params_prf'{algorithm = ?'id-hmacWithSHA1'}) -> + {fun crypto:sha_mac/3, pseudo_output_length(?'id-hmacWithSHA1')}. + +pseudo_output_length(?'id-hmacWithSHA1') -> + ?DEFAULT_SHA_MAC_KEYLEN. + +derived_key_length(_, Len) when is_integer(Len) -> + Len; +derived_key_length(Cipher,_) when (Cipher == ?'desCBC') or + (Cipher == "DES-CBC") -> + 8; +derived_key_length(Cipher,_) when (Cipher == ?'rc2CBC') or + (Cipher == "RC2-CBC") -> + 16; +derived_key_length(Cipher,_) when (Cipher == ?'des-EDE3-CBC') or + (Cipher == "DES-EDE3-CBC") -> + 24. + +cipher(#'PBES2-params_encryptionScheme'{algorithm = ?'desCBC'}) -> + "DES-CBC"; +cipher(#'PBES2-params_encryptionScheme'{algorithm = ?'des-EDE3-CBC'}) -> + "DES-EDE3-CBC"; +cipher(#'PBES2-params_encryptionScheme'{algorithm = ?'rc2CBC'}) -> + "RC2-CBC". + +ceiling(Float) -> + erlang:round(Float + 0.5). diff --git a/lib/public_key/src/pubkey_pem.erl b/lib/public_key/src/pubkey_pem.erl index c26815bc04..910473d629 100644 --- a/lib/public_key/src/pubkey_pem.erl +++ b/lib/public_key/src/pubkey_pem.erl @@ -43,8 +43,6 @@ -include("public_key.hrl"). -export([encode/1, decode/1, decipher/2, cipher/3]). -%% Backwards compatibility --export([decode_key/2]). -define(ENCODED_LINE_LENGTH, 64). @@ -69,23 +67,23 @@ encode(PemEntries) -> encode_pem_entries(PemEntries). %%-------------------------------------------------------------------- --spec decipher({pki_asn1_type(), DerEncrypted::binary(),{Cipher :: string(), - Salt :: binary()}}, +-spec decipher({pki_asn1_type(), DerEncrypted::binary(), + {Cipher :: string(), Salt :: iodata() | #'PBES2-params'{}}}, string()) -> Der::binary(). %% %% Description: Deciphers a decrypted pem entry. %%-------------------------------------------------------------------- -decipher({_, DecryptDer, {Cipher,Salt}}, Password) -> - decode_key(DecryptDer, Password, Cipher, Salt). +decipher({_, DecryptDer, {Cipher, KeyDevParams}}, Password) -> + pubkey_pbe:decode(DecryptDer, Password, Cipher, KeyDevParams). %%-------------------------------------------------------------------- --spec cipher(Der::binary(),{Cipher :: string(), Salt :: binary()} , +-spec cipher(Der::binary(), {Cipher :: string(), Salt :: iodata() | #'PBES2-params'{}} , string()) -> binary(). %% %% Description: Ciphers a PEM entry %%-------------------------------------------------------------------- -cipher(Der, {Cipher,Salt}, Password)-> - encode_key(Der, Password, Cipher, Salt). +cipher(Der, {Cipher, KeyDevParams}, Password)-> + pubkey_pbe:encode(Der, Password, Cipher, KeyDevParams). %%-------------------------------------------------------------------- %%% Internal functions @@ -127,8 +125,20 @@ decode_pem_entry(Start, Lines) -> Type = asn1_type(Start), Cs = erlang:iolist_to_binary(Lines), Decoded = base64:mime_decode(Cs), - {Type, Decoded, not_encrypted}. - + case Type of + 'EncryptedPrivateKeyInfo'-> + decode_encrypted_private_keyinfo(Decoded); + _ -> + {Type, Decoded, not_encrypted} + end. + +decode_encrypted_private_keyinfo(Der) -> + #'EncryptedPrivateKeyInfo'{encryptionAlgorithm = AlgorithmInfo, + encryptedData = Data} = + public_key:der_decode('EncryptedPrivateKeyInfo', Der), + DecryptParams = pubkey_pbe:decrypt_parameters(AlgorithmInfo), + {'PrivateKeyInfo', iolist_to_binary(Data), DecryptParams}. + split_bin(Bin) -> split_bin(0, Bin). @@ -160,37 +170,6 @@ join_entry([<<"-----END ", _/binary>>| Lines], Entry) -> join_entry([Line | Lines], Entry) -> join_entry(Lines, [Line | Entry]). -decode_key(Data, Password, "DES-CBC", Salt) -> - Key = password_to_key(Password, Salt, 8), - IV = Salt, - crypto:des_cbc_decrypt(Key, IV, Data); -decode_key(Data, Password, "DES-EDE3-CBC", Salt) -> - Key = password_to_key(Password, Salt, 24), - IV = Salt, - <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, - crypto:des_ede3_cbc_decrypt(Key1, Key2, Key3, IV, Data). - -encode_key(Data, Password, "DES-CBC", Salt) -> - Key = password_to_key(Password, Salt, 8), - IV = Salt, - crypto:des_cbc_encrypt(Key, IV, Data); -encode_key(Data, Password, "DES-EDE3-CBC", Salt) -> - Key = password_to_key(Password, Salt, 24), - IV = Salt, - <<Key1:8/binary, Key2:8/binary, Key3:8/binary>> = Key, - crypto:des_ede3_cbc_encrypt(Key1, Key2, Key3, IV, Data). - -password_to_key(Data, Salt, KeyLen) -> - <<Key:KeyLen/binary, _/binary>> = - password_to_key(<<>>, Data, Salt, KeyLen, <<>>), - Key. - -password_to_key(_, _, _, Len, Acc) when Len =< 0 -> - Acc; -password_to_key(Prev, Data, Salt, Len, Acc) -> - M = crypto:md5([Prev, Data, Salt]), - password_to_key(M, Data, Salt, Len - size(M), <<Acc/binary, M/binary>>). - unhex(S) -> unhex(S, []). @@ -228,6 +207,10 @@ pem_end(<<"-----BEGIN DSA PRIVATE KEY-----">>) -> <<"-----END DSA PRIVATE KEY-----">>; pem_end(<<"-----BEGIN DH PARAMETERS-----">>) -> <<"-----END DH PARAMETERS-----">>; +pem_end(<<"-----BEGIN PRIVATE KEY-----">>) -> + <<"-----END PRIVATE KEY-----">>; +pem_end(<<"-----BEGIN ENCRYPTED PRIVATE KEY-----">>) -> + <<"-----END ENCRYPTED PRIVATE KEY-----">>; pem_end(_) -> undefined. @@ -242,18 +225,14 @@ asn1_type(<<"-----BEGIN PUBLIC KEY-----">>) -> asn1_type(<<"-----BEGIN DSA PRIVATE KEY-----">>) -> 'DSAPrivateKey'; asn1_type(<<"-----BEGIN DH PARAMETERS-----">>) -> - 'DHParameter'. + 'DHParameter'; +asn1_type(<<"-----BEGIN PRIVATE KEY-----">>) -> + 'PrivateKeyInfo'; +asn1_type(<<"-----BEGIN ENCRYPTED PRIVATE KEY-----">>) -> + 'EncryptedPrivateKeyInfo'. pem_decrypt() -> <<"Proc-Type: 4,ENCRYPTED">>. pem_decrypt_info(Cipher, Salt) -> io_lib:format("DEK-Info: ~s,~s", [Cipher, lists:flatten(hexify(Salt))]). - -%%-------------------------------------------------------------------- -%%% Deprecated -%%-------------------------------------------------------------------- -decode_key({_Type, Bin, not_encrypted}, _) -> - Bin; -decode_key({_Type, Bin, {Chipher,Salt}}, Password) -> - decode_key(Bin, Password, Chipher, Salt). diff --git a/lib/public_key/src/public_key.app.src b/lib/public_key/src/public_key.app.src index 1963bd05d4..4cc81ea573 100644 --- a/lib/public_key/src/public_key.app.src +++ b/lib/public_key/src/public_key.app.src @@ -3,10 +3,12 @@ {vsn, "%VSN%"}, {modules, [ public_key, pubkey_pem, + pubkey_pbe, pubkey_ssh, pubkey_cert, pubkey_cert_records, - 'OTP-PUB-KEY' + 'OTP-PUB-KEY', + 'PKCS-FRAME' ]}, {applications, [crypto, kernel, stdlib]}, {registered, []}, diff --git a/lib/public_key/src/public_key.appup.src b/lib/public_key/src/public_key.appup.src index 4986801dad..2945fb1213 100644 --- a/lib/public_key/src/public_key.appup.src +++ b/lib/public_key/src/public_key.appup.src @@ -1,70 +1,16 @@ %% -*- erlang -*- {"%VSN%", [ - {"0.11", - [ - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_pem, soft, soft_purge, soft_purge, []}, - {add_module, pubkey_ssh, soft, soft_purge, soft_purge}, - {update, pubkey_cert, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert_records, soft, soft_purge, soft_purge, []} - ] - }, - - {"0.10", - [ - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_pem, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert_records, soft, soft_purge, soft_purge, []} - ] - }, - {"0.9", - [ - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert, soft, soft_purge, soft_purge, []} - ] - }, - {"0.8", - [ - {update, 'OTP-PUB-KEY', soft, soft_purge, soft_purge, []}, - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_pem, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert_records, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert, soft, soft_purge, soft_purge, []} - ] - } + {"0.13", [{restart_application, public_key}]}, + {"0.11", [{restart_application, public_key}]}, + {"0.10", [{restart_application, public_key}]}, + {"0.9", [{restart_application, public_key}]}, + {"0.8", [{restart_application, public_key}]} ], [ - {"0.11", - [ - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_pem, soft, soft_purge, soft_purge, []}, - {delete_module, pubkey_ssh, soft, soft_purge, soft_purge}, - {update, pubkey_cert, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert_records, soft, soft_purge, soft_purge, []} - ] - }, - - {"0.10", - [ - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_pem, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert_records, soft, soft_purge, soft_purge, []} - ] - }, - {"0.9", - [ - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert, soft, soft_purge, soft_purge, []} - ] - }, - {"0.8", - [ - {update, 'OTP-PUB-KEY', soft, soft_purge, soft_purge, []}, - {update, public_key, soft, soft_purge, soft_purge, []}, - {update, pubkey_pem, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert_records, soft, soft_purge, soft_purge, []}, - {update, pubkey_cert, soft, soft_purge, soft_purge, []} - ] - } + {"0.13", [{restart_application, public_key}]}, + {"0.11", [{restart_application, public_key}]}, + {"0.10", [{restart_application, public_key}]}, + {"0.9", [{restart_application, public_key}]}, + {"0.8", [{restart_application, public_key}]} ]}. diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index 33fcce2c44..753322b46d 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -45,13 +45,6 @@ ssh_decode/2, ssh_encode/2 ]). -%% Deprecated --export([decode_private_key/1, decode_private_key/2, pem_to_der/1]). - --deprecated({pem_to_der, 1, next_major_release}). --deprecated({decode_private_key, 1, next_major_release}). --deprecated({decode_private_key, 2, next_major_release}). - -type rsa_padding() :: 'rsa_pkcs1_padding' | 'rsa_pkcs1_oaep_padding' | 'rsa_no_padding'. -type public_crypt_options() :: [{rsa_pad, rsa_padding()}]. @@ -104,22 +97,23 @@ pem_entry_decode({Asn1Type, Der, not_encrypted}) when is_atom(Asn1Type), pem_entry_decode({Asn1Type, Der, not_encrypted}, _) when is_atom(Asn1Type), is_binary(Der) -> der_decode(Asn1Type, Der); +pem_entry_decode({Asn1Type, CryptDer, {Cipher, #'PBES2-params'{}}} = PemEntry, + Password) when is_atom(Asn1Type) andalso + is_binary(CryptDer) andalso + is_list(Cipher) -> + do_pem_entry_decode(PemEntry, Password); pem_entry_decode({Asn1Type, CryptDer, {Cipher, Salt}} = PemEntry, - Password) when is_atom(Asn1Type), - is_binary(CryptDer), - is_list(Cipher), - is_binary(Salt), - erlang:byte_size(Salt) == 8 - -> - Der = pubkey_pem:decipher(PemEntry, Password), - der_decode(Asn1Type, Der). + Password) when is_atom(Asn1Type) andalso + is_binary(CryptDer) andalso + is_list(Cipher) andalso + is_binary(Salt) andalso + erlang:byte_size(Salt) == 8 -> + do_pem_entry_decode(PemEntry, Password). %%-------------------------------------------------------------------- -spec pem_entry_encode(pki_asn1_type(), term()) -> pem_entry(). --spec pem_entry_encode(pki_asn1_type(), term(), - {{Cipher :: string(), Salt :: binary()}, string()}) -> - pem_entry(). -% +-spec pem_entry_encode(pki_asn1_type(), term(), term()) -> pem_entry(). +%% %% Description: Creates a pem entry that can be feed to pem_encode/1. %%-------------------------------------------------------------------- pem_entry_encode('SubjectPublicKeyInfo', Entity=#'RSAPublicKey'{}) -> @@ -137,21 +131,36 @@ pem_entry_encode('SubjectPublicKeyInfo', pem_entry_encode(Asn1Type, Entity) when is_atom(Asn1Type) -> Der = der_encode(Asn1Type, Entity), {Asn1Type, Der, not_encrypted}. -pem_entry_encode(Asn1Type, Entity, - {{Cipher, Salt}= CipherInfo, Password}) when is_atom(Asn1Type), - is_list(Cipher), - is_binary(Salt), - erlang:byte_size(Salt) == 8, - is_list(Password)-> - Der = der_encode(Asn1Type, Entity), - DecryptDer = pubkey_pem:cipher(Der, CipherInfo, Password), - {Asn1Type, DecryptDer, CipherInfo}. - +pem_entry_encode(Asn1Type, Entity, {{Cipher, #'PBES2-params'{}} = CipherInfo, + Password}) when is_atom(Asn1Type) andalso + is_list(Password) andalso + is_list(Cipher) -> + do_pem_entry_encode(Asn1Type, Entity, CipherInfo, Password); + +pem_entry_encode(Asn1Type, Entity, {{Cipher, Salt} = CipherInfo, + Password}) when is_atom(Asn1Type) andalso + is_list(Password) andalso + is_list(Cipher) andalso + is_binary(Salt) andalso + erlang:byte_size(Salt) == 8 -> + do_pem_entry_encode(Asn1Type, Entity, CipherInfo, Password). + %%-------------------------------------------------------------------- -spec der_decode(asn1_type(), Der::binary()) -> term(). %% %% Description: Decodes a public key asn1 der encoded entity. %%-------------------------------------------------------------------- +der_decode(Asn1Type, Der) when (Asn1Type == 'PrivateKeyInfo') or + (Asn1Type == 'EncryptedPrivateKeyInfo') + andalso is_binary(Der) -> + try + {ok, Decoded} = 'PKCS-FRAME':decode(Asn1Type, Der), + Decoded + catch + error:{badmatch, {error, _}} = Error -> + erlang:error(Error) + end; + der_decode(Asn1Type, Der) when is_atom(Asn1Type), is_binary(Der) -> try {ok, Decoded} = 'OTP-PUB-KEY':decode(Asn1Type, Der), @@ -166,6 +175,16 @@ der_decode(Asn1Type, Der) when is_atom(Asn1Type), is_binary(Der) -> %% %% Description: Encodes a public key entity with asn1 DER encoding. %%-------------------------------------------------------------------- +der_encode(Asn1Type, Entity) when (Asn1Type == 'PrivateKeyInfo') or + (Asn1Type == 'EncryptedPrivateKeyInfo') -> + try + {ok, Encoded} = 'PKCS-FRAME':encode(Asn1Type, Entity), + iolist_to_binary(Encoded) + catch + error:{badmatch, {error, _}} = Error -> + erlang:error(Error) + end; + der_encode(Asn1Type, Entity) when is_atom(Asn1Type) -> try {ok, Encoded} = 'OTP-PUB-KEY':encode(Asn1Type, Entity), @@ -535,6 +554,14 @@ ssh_encode(Entries, Type) when is_list(Entries), %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- +do_pem_entry_encode(Asn1Type, Entity, CipherInfo, Password) -> + Der = der_encode(Asn1Type, Entity), + DecryptDer = pubkey_pem:cipher(Der, CipherInfo, Password), + {Asn1Type, DecryptDer, CipherInfo}. + +do_pem_entry_decode({Asn1Type,_, _} = PemEntry, Password) -> + Der = pubkey_pem:decipher(PemEntry, Password), + der_decode(Asn1Type, Der). encrypt_public(PlainText, N, E, Options)-> Padding = proplists:get_value(rsa_pad, Options, rsa_pkcs1_padding), @@ -632,20 +659,3 @@ validate(DerCert, #path_validation_state{working_issuer_name = Issuer, sized_binary(Binary) -> Size = size(Binary), <<?UINT32(Size), Binary/binary>>. - -%%-------------------------------------------------------------------- -%%% Deprecated functions -%%-------------------------------------------------------------------- -pem_to_der(CertSource) -> - {ok, Bin} = file:read_file(CertSource), - {ok, pubkey_pem:decode(Bin)}. - -decode_private_key(KeyInfo) -> - decode_private_key(KeyInfo, no_passwd). - -decode_private_key(KeyInfo = {'RSAPrivateKey', _, _}, Password) -> - DerEncoded = pubkey_pem:decode_key(KeyInfo, Password), - 'OTP-PUB-KEY':decode('RSAPrivateKey', DerEncoded); -decode_private_key(KeyInfo = {'DSAPrivateKey', _, _}, Password) -> - DerEncoded = pubkey_pem:decode_key(KeyInfo, Password), - 'OTP-PUB-KEY':decode('DSAPrivateKey', DerEncoded). diff --git a/lib/public_key/test/Makefile b/lib/public_key/test/Makefile index 6889ae9a8a..b7f91981a5 100644 --- a/lib/public_key/test/Makefile +++ b/lib/public_key/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2010. All Rights Reserved. +# Copyright Ericsson AB 2008-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -30,6 +30,7 @@ INCLUDES= -I. -I ../include MODULES= \ erl_make_certs \ public_key_SUITE \ + pbe_SUITE \ pkits_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/public_key/test/pbe_SUITE.erl b/lib/public_key/test/pbe_SUITE.erl new file mode 100644 index 0000000000..380a67db7b --- /dev/null +++ b/lib/public_key/test/pbe_SUITE.erl @@ -0,0 +1,259 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2011-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(pbe_SUITE). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +%% Note: This directive should only be used in test suites. +-compile(export_all). +%% Test server callback functions +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config) -> Config +%% Config - [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Description: Initialization before the whole suite +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + try crypto:start() of + ok -> + Config + catch _:_ -> + {skip, "Crypto did not start"} + end. +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config) -> _ +%% Config - [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Description: Cleanup after the whole suite +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + application:stop(crypto). + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config) -> Config +%% Case - atom() +%% Name of the test case that is about to be run. +%% Config - [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Initialization before each test case +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%% Description: Initialization before each test case +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config) -> _ +%% Case - atom() +%% Name of the test case that is about to be run. +%% Config - [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Description: Cleanup after each test case +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: all(Clause) -> TestCases +%% Clause - atom() - suite | doc +%% TestCases - [Case] +%% Case - atom() +%% Name of a test case. +%% Description: Returns a list of all test cases in this test suite +%%-------------------------------------------------------------------- +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + pbdkdf1, + pbdkdf2, + encrypted_private_key_info]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +%% Test cases starts here. +%%-------------------------------------------------------------------- +pbdkdf1(doc) -> + ["Test with PKCS #5 PBKDF1 Test Vectors"]; +pbdkdf1(Config) when is_list(Config) -> + %%Password = "password" + %% = (0x)70617373776F7264 + %%Salt = (0x)78578E5A5D63CB06 + %%Count = 1000 + %%kLen = 16 + %%Key = PBKDF1(Password, Salt, Count, kLen) + %%= (0x)DC19847E05C64D2FAF10EBFB4A3D2A20 + + Password = "password", + Salt = <<16#78,16#57,16#8E,16#5A,16#5D,16#63,16#CB,16#06>>, + Count = 1000, + + <<16#DC, 16#19, 16#84, 16#7E, + 16#05, 16#C6, 16#4D, 16#2F, + 16#AF, 16#10, 16#EB, 16#FB, + 16#4A, 16#3D, 16#2A, 16#20, _/binary>> = + pubkey_pbe:pbdkdf1(Password, Salt, Count, sha). + +pbdkdf2(doc) -> + ["Test with PKCS #5 PBKDF2 Test Vectors"]; +pbdkdf2(Config) when is_list(Config) -> + %% Input: + %% P = "password" (8 octets) + %% S = "salt" (4 octets) + %% c = 1 + %% dkLen = 20 + + %% Output: + %% DK = 0c 60 c8 0f 96 1f 0e 71 + %% f3 a9 b5 24 af 60 12 06 + %% 2f e0 37 a6 (20 octets) + + <<16#0c, 16#60, 16#c8, 16#0f, 16#96, 16#1f, 16#0e, 16#71, + 16#f3, 16#a9, 16#b5, 16#24, 16#af, 16#60, 16#12, 16#06, + 16#2f, 16#e0, 16#37, 16#a6>> = pubkey_pbe:pbdkdf2("password", "salt", 1, 20, fun crypto:sha_mac/3, 20), + + %% Input: + %% P = "password" (8 octets) + %% S = "salt" (4 octets) + %% c = 2 + %% dkLen = 20 + + %% Output: + %% DK = ea 6c 01 4d c7 2d 6f 8c + %% cd 1e d9 2a ce 1d 41 f0 + %% d8 de 89 57 (20 octets) + + <<16#ea, 16#6c, 16#01, 16#4d, 16#c7, 16#2d, 16#6f, 16#8c, + 16#cd, 16#1e, 16#d9, 16#2a, 16#ce, 16#1d, 16#41, 16#f0, + 16#d8, 16#de, 16#89, 16#57>> = + pubkey_pbe:pbdkdf2("password", "salt", 2, 20, fun crypto:sha_mac/3, 20), + + %% Input: + %% P = "password" (8 octets) + %% S = "salt" (4 octets) + %% c = 4096 + %% dkLen = 20 + + %% Output: + %% DK = 4b 00 79 01 b7 65 48 9a + %% be ad 49 d9 26 f7 21 d0 + %% 65 a4 29 c1 (20 octets) + + <<16#4b, 16#00, 16#79, 16#01, 16#b7, 16#65, 16#48, 16#9a, + 16#be, 16#ad, 16#49, 16#d9, 16#26, 16#f7, 16#21, 16#d0, + 16#65, 16#a4, 16#29, 16#c1>> = pubkey_pbe:pbdkdf2("password", "salt", 4096, 20, fun crypto:sha_mac/3, 20), + + %% Input: + %% P = "password" (8 octets) + %% S = "salt" (4 octets) + %% c = 16777216 + %% dkLen = 20 + + %% Output: + %% DK = ee fe 3d 61 cd 4d a4 e4 + %% e9 94 5b 3d 6b a2 15 8c + %% 26 34 e9 84 (20 octets) + + + <<16#ee, 16#fe, 16#3d, 16#61, 16#cd, 16#4d, 16#a4, 16#e4, + 16#e9, 16#94, 16#5b, 16#3d, 16#6b, 16#a2, 16#15, 16#8c, + 16#26, 16#34, 16#e9, 16#84>> = pubkey_pbe:pbdkdf2("password", "salt", 16777216, 20, fun crypto:sha_mac/3, 20), + + %% Input: + %% P = "passwordPASSWORDpassword" (24 octets) + %% S = "saltSALTsaltSALTsaltSALTsaltSALTsalt" (36 octets) + %% c = 4096 + %% dkLen = 25 + + %% Output: + %% DK = 3d 2e ec 4f e4 1c 84 9b + %% 80 c8 d8 36 62 c0 e4 4a + %% 8b 29 1a 96 4c f2 f0 70 + %% 38 (25 octets) + + <<16#3d, 16#2e, 16#ec, 16#4f, 16#e4, 16#1c, 16#84, 16#9b, + 16#80, 16#c8, 16#d8, 16#36, 16#62, 16#c0, 16#e4, 16#4a, + 16#8b, 16#29, 16#1a, 16#96, 16#4c, 16#f2, 16#f0, 16#70, + 16#38>> + = pubkey_pbe:pbdkdf2("passwordPASSWORDpassword", + "saltSALTsaltSALTsaltSALTsaltSALTsalt", 4096, 25, fun crypto:sha_mac/3, 20), + + %% Input: + %% P = "pass\0word" (9 octets) + %% S = "sa\0lt" (5 octets) + %% c = 4096 + %% dkLen = 16 + + %% Output: + %% DK = 56 fa 6a a7 55 48 09 9d + %% cc 37 d7 f0 34 25 e0 c3 (16 octets) + + <<16#56, 16#fa, 16#6a, 16#a7, 16#55, 16#48, 16#09, 16#9d, + 16#cc, 16#37, 16#d7, 16#f0, 16#34, 16#25, 16#e0, 16#c3>> + = pubkey_pbe:pbdkdf2("pass\0word", + "sa\0lt", 4096, 16, fun crypto:sha_mac/3, 20). + +encrypted_private_key_info(doc) -> + ["Tests reading a EncryptedPrivateKeyInfo file encrypted with different ciphers"]; +encrypted_private_key_info(Config) when is_list(Config) -> + Datadir = ?config(data_dir, Config), + {ok, PemDes} = file:read_file(filename:join(Datadir, "des_cbc_enc_key.pem")), + + PemDesEntry = public_key:pem_decode(PemDes), + test_server:format("Pem entry: ~p" , [PemDesEntry]), + [{'PrivateKeyInfo', _, {"DES-CBC",_}} = PubEntry0] = PemDesEntry, + KeyInfo = public_key:pem_entry_decode(PubEntry0, "password"), + + {ok, Pem3Des} = file:read_file(filename:join(Datadir, "des_ede3_cbc_enc_key.pem")), + + Pem3DesEntry = public_key:pem_decode(Pem3Des), + test_server:format("Pem entry: ~p" , [Pem3DesEntry]), + [{'PrivateKeyInfo', _, {"DES-EDE3-CBC",_}} = PubEntry1] = Pem3DesEntry, + KeyInfo = public_key:pem_entry_decode(PubEntry1, "password"), + + {ok, PemRc2} = file:read_file(filename:join(Datadir, "rc2_cbc_enc_key.pem")), + + PemRc2Entry = public_key:pem_decode(PemRc2), + test_server:format("Pem entry: ~p" , [PemRc2Entry]), + [{'PrivateKeyInfo', _, {"RC2-CBC",_}} = PubEntry2] = PemRc2Entry, + KeyInfo = public_key:pem_entry_decode(PubEntry2, "password"), + + check_key_info(KeyInfo). + + +check_key_info(#'PrivateKeyInfo'{privateKeyAlgorithm = + #'PrivateKeyInfo_privateKeyAlgorithm'{algorithm = ?rsaEncryption}, + privateKey = Key}) -> + #'RSAPrivateKey'{} = public_key:der_decode('RSAPrivateKey', iolist_to_binary(Key)). diff --git a/lib/public_key/test/pbe_SUITE_data/des_cbc_enc_key.pem b/lib/public_key/test/pbe_SUITE_data/des_cbc_enc_key.pem new file mode 100644 index 0000000000..eaa06145aa --- /dev/null +++ b/lib/public_key/test/pbe_SUITE_data/des_cbc_enc_key.pem @@ -0,0 +1,11 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIBozA9BgkqhkiG9w0BBQ0wMDAbBgkqhkiG9w0BBQwwDgQIfWBDXwLp4K4CAggA +MBEGBSsOAwIHBAiaCF/AvOgQ6QSCAWDWX4BdAzCRNSQSANSuNsT5X8mWYO27mr3Y +9c9LoBVXGNmYWKA77MI4967f7SmjNcgXj3xNE/jmnVz6hhsjS8E5VPT3kfyVkpdZ +0lr5e9Yk2m3JWpPU7++v5zBkZmC4V/MwV/XuIs6U+vykgzMgpxQg0oZKS9zgmiZo +f/4dOCL0UtCDnyOSvqT7mCVIcMDIEKu8QbVlgZYBop08l60EuEU3gARUo8WsYQmO +Dz/ldx0Z+znIT0SXVuOwc+RVItC5T/Qx+aijmmpt+9l14nmaGBrEkmuhmtdvU/4v +aptewGRgmjOfD6cqK+zs0O5NrrJ3P/6ZSxXj91CQgrThGfOv72bUncXEMNtc8pks +2jpHFjGMdKufnadAD7XuMgzkkaklEXZ4f5tU6heIIwr51g0GBEGF96gYPFnjnSQM +75JE02Clo+DfcfXpcybPTwwFg2jd6JTTOfkdf6OdSlA/1XNK43FA +-----END ENCRYPTED PRIVATE KEY----- diff --git a/lib/public_key/test/pbe_SUITE_data/des_ede3_cbc_enc_key.pem b/lib/public_key/test/pbe_SUITE_data/des_ede3_cbc_enc_key.pem new file mode 100644 index 0000000000..22ea46d56f --- /dev/null +++ b/lib/public_key/test/pbe_SUITE_data/des_ede3_cbc_enc_key.pem @@ -0,0 +1,11 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIBpjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQIeFeOWl1jywYCAggA +MBQGCCqGSIb3DQMHBAjUJ5eGBhQGtQSCAWBrHrRgqO8UUMLcWzZEtpk1l3mjxiF/ +koCMkHsFwowgyWhEbgIkTgbSViK54LVK8PskekcGNLph+rB6bGZ7pPbL5pbXASJ8 ++MkQcG3FZdlS4Ek9tTJDApj3O1UubZGFG4uvTlJJFbF1BOJ3MkY3XQ9Gl1qwv7j5 +6e103Da7Cq9+oIDKmznza78XXQYrUsPo8mJGjUxPskEYlzwvHjKubRnYm/K6RKhi +5f4zX4BQ/Dt3H812ZjRXrsjAJP0KrD/jyD/jCT7zNBVPH1izBds+RwizyQAHwfNJ +BFR78TH4cgzB619X47FDVOnT0LqQNVd0O3cSwnPrXE9XR3tPayE+iOB15llFSmi8 +z0ByOXldEpkezCn92Umk++suzIVj1qfsK+bv2phZWJPbLEIWPDRHUbYf76q5ArAr +u4xtxT/hoK3krEs/IN3d70qjlUJ36SEw1UaZ82PWhakQbdtu39ZraMJB +-----END ENCRYPTED PRIVATE KEY----- diff --git a/lib/public_key/test/pbe_SUITE_data/rc2_cbc_enc_key.pem b/lib/public_key/test/pbe_SUITE_data/rc2_cbc_enc_key.pem new file mode 100644 index 0000000000..618cddcfd7 --- /dev/null +++ b/lib/public_key/test/pbe_SUITE_data/rc2_cbc_enc_key.pem @@ -0,0 +1,12 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIBrjBIBgkqhkiG9w0BBQ0wOzAeBgkqhkiG9w0BBQwwEQQIrHyQPBZqWLUCAggA +AgEQMBkGCCqGSIb3DQMCMA0CAToECEhbh7YZKiPSBIIBYCT1zp6o5jpFlIkgwPop +7bW1+8ACr4exqzkeb3WflQ8cWJ4cURxzVdvxUnXeW1VJdaQZtjS/QHs5GhPTG/0f +wtvnaPfwrIJ3FeGaZfcg2CrYhalOFmEb4xrE4KyoEQmUN8tb/Cg94uzd16BOPw21 +RDnE8bnPdIGY7TyL95kbkqH23mK53pi7h+xWIgduW+atIqDyyt55f7WMZcvDvlj6 +VpN/V0h+qxBHL274WA4dj6GYgeyUFpi60HdGCK7By2TBy8h1ZvKGjmB9h8jZvkx1 +MkbRumXxyFsowTZawyYvO8Um6lbfEDP9zIEUq0IV8RqH2MRyblsPNSikyYhxX/cz +tdDxRKhilySbSBg5Kr8OfcwKp9bpinN96nmG4xr3Tch1bnVvqJzOQ5+Vva2WwVvH +2JkWvYm5WaANg4Q6bRxu9vz7DuhbJjQdZbxFezIAgrJdSe92B00jO/0Kny1WjiVO +6DA= +-----END ENCRYPTED PRIVATE KEY----- diff --git a/lib/public_key/test/public_key_SUITE.erl b/lib/public_key/test/public_key_SUITE.erl index b11e4d092a..a91dcfa029 100644 --- a/lib/public_key/test/public_key_SUITE.erl +++ b/lib/public_key/test/public_key_SUITE.erl @@ -23,8 +23,8 @@ %% Note: This directive should only be used in test suites. -compile(export_all). --include_lib("common_test/include/ct.hrl"). --include_lib("test_server/include/test_server_line.hrl"). +%%-include_lib("common_test/include/ct.hrl"). +-include_lib("test_server/include/test_server.hrl"). -include_lib("public_key/include/public_key.hrl"). @@ -107,7 +107,7 @@ all() -> {group, ssh_public_key_decode_encode}, encrypt_decrypt, {group, sign_verify}, - pkix, pkix_path_validation, deprecated]. + pkix, pkix_path_validation]. groups() -> [{pem_decode_encode, [], [dsa_pem, rsa_pem, encrypted_pem, @@ -699,27 +699,6 @@ pkix_path_validation(Config) when is_list(Config) -> VerifyFunAndState1}]), ok. -%%-------------------------------------------------------------------- -deprecated(doc) -> - ["Check deprecated functions."]; -deprecated(suite) -> - []; -deprecated(Config) when is_list(Config) -> - Datadir = ?config(data_dir, Config), - {ok, [DsaKey = {'DSAPrivateKey', _DsaKey, _}]} = - public_key:pem_to_der(filename:join(Datadir, "dsa.pem")), - {ok, [RsaKey = {'RSAPrivateKey', _RsaKey,_}]} = - public_key:pem_to_der(filename:join(Datadir, "client_key.pem")), - {ok, [ProtectedRsaKey = {'RSAPrivateKey', _ProtectedRsaKey,_}]} = - public_key:pem_to_der(filename:join(Datadir, "rsa.pem")), - - {ok, #'DSAPrivateKey'{}} = public_key:decode_private_key(DsaKey), - {ok, #'RSAPrivateKey'{}} = public_key:decode_private_key(RsaKey), - {ok, #'RSAPrivateKey'{}} = public_key:decode_private_key(ProtectedRsaKey, "abcd1234"), - ok. - -%%-------------------------------------------------------------------- - check_entry_type(#'DSAPrivateKey'{}, 'DSAPrivateKey') -> true; check_entry_type(#'RSAPrivateKey'{}, 'RSAPrivateKey') -> diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 66ac78a65d..d8f811bf25 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 0.13 +PUBLIC_KEY_VSN = 0.14 diff --git a/lib/runtime_tools/src/erts_alloc_config.erl b/lib/runtime_tools/src/erts_alloc_config.erl index 0bcb202fd8..1a57c94443 100644 --- a/lib/runtime_tools/src/erts_alloc_config.erl +++ b/lib/runtime_tools/src/erts_alloc_config.erl @@ -1,6 +1,7 @@ +%% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2009. All Rights Reserved. +%% Copyright Ericsson AB 2007-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,6 +17,7 @@ %% The Initial Developer of the Original Code is Ericsson AB. %% %% %CopyrightEnd% +%% %%%------------------------------------------------------------------- %%% File : erts_alloc_config.erl @@ -71,6 +73,7 @@ A == binary_alloc; A == std_alloc; A == ets_alloc; + A == fix_alloc; A == eheap_alloc; A == ll_alloc; A == sl_alloc; @@ -94,6 +97,7 @@ [{binary_alloc, 131072}, {std_alloc, 131072}, {ets_alloc, 131072}, + {fix_alloc, 131072}, {eheap_alloc, 524288}, {ll_alloc, 2097152}, {sl_alloc, 131072}, @@ -104,6 +108,7 @@ [{binary_alloc, 10}, {std_alloc, 10}, {ets_alloc, 10}, + {fix_alloc, 10}, {eheap_alloc, 10}, {ll_alloc, 0}, {sl_alloc, 10}, @@ -438,9 +443,6 @@ conf_alloc(#conf{format_to = FTO} = Conf, #alloc{name = A} = Alc) -> chk_xnote(Conf, Alc). chk_xnote(#conf{format_to = FTO}, - #alloc{name = fix_alloc}) -> - fcp(FTO, "Cannot be configured."); -chk_xnote(#conf{format_to = FTO}, #alloc{name = sys_alloc}) -> fcp(FTO, "Cannot be configured. Default malloc implementation used."); chk_xnote(#conf{format_to = FTO}, diff --git a/lib/sasl/test/release_handler_SUITE_data/heart_restart.bat b/lib/sasl/test/release_handler_SUITE_data/heart_restart.bat index ede1ad4ff3..ede1ad4ff3 100755..100644 --- a/lib/sasl/test/release_handler_SUITE_data/heart_restart.bat +++ b/lib/sasl/test/release_handler_SUITE_data/heart_restart.bat diff --git a/lib/snmp/doc/src/notes.xml b/lib/snmp/doc/src/notes.xml index 9e1a060dee..fd5a548b16 100644 --- a/lib/snmp/doc/src/notes.xml +++ b/lib/snmp/doc/src/notes.xml @@ -32,6 +32,120 @@ <file>notes.xml</file> </header> + + <section> + <title>SNMP Development Toolkit 4.21.3</title> + <p>Version 4.21.3 supports code replacement in runtime from/to + version 4.21.2, 4.21.1, 4.21 and 4.20.1. </p> + + <section> + <title>Improvements and new features</title> +<!-- + <p>-</p> +--> + + <list type="bulleted"> + <item> + <p>[compiler] Improved version info printout. </p> + <p>Own Id: OTP-9618</p> + </item> + + </list> + </section> + + <section> + <title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + + <list type="bulleted"> + <item> + <p>[compiler] Fix the <c>--warnings/--W</c> option parsing in the + <seealso marker="snmpc(command)#option_warnings">snmpc</seealso> + wrapper (e)script. + The short warning option was incorrectly <c>--w</c>, instead + of as documented <c>--W</c>. This has now been corrected. </p> + <p>*** POTENTIAL INCOMPATIBILITY ***</p> + <p>Tuncer Ayaz</p> + <p>Own Id: OTP-9718</p> + </item> + + </list> + + </section> + + + <section> + <title>Incompatibilities</title> +<!-- + <p>-</p> +--> + + <list type="bulleted"> + <item> + <p>[compiler] The short warning option has been changed from + <c>--w</c> to <c>--W</c> to comply with the documentation. </p> + <p>Tuncer Ayaz</p> + <p>Own Id: OTP-9718</p> + </item> + + + </list> + </section> + + </section> <!-- 4.21.3 --> + + + <section> + <title>SNMP Development Toolkit 4.21.2</title> + <p>Version 4.21.2 supports code replacement in runtime from/to + version 4.21.1, 4.21, 4.20.1, 4.20 and 4.19. </p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + +<!-- + <list type="bulleted"> + <item> + <p>Bad note store GC timer deactivation. + Wrong field in the state record was set (timeout instead active). </p> + <p>Stefan Grundmann</p> + <p>Own Id: OTP-9690</p> + </item> + + </list> +--> + + </section> + + <section> + <title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + + <list type="bulleted"> + <item> + <p>Bad note store GC timer deactivation. + Wrong field in the state record was set (timeout instead active). </p> + <p>Stefan Grundmann</p> + <p>Own Id: OTP-9690</p> + </item> + + </list> + </section> + + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + + </section> <!-- 4.21.2 --> + + <section> <title>SNMP Development Toolkit 4.21.1</title> <p>Version 4.21.1 supports code replacement in runtime from/to @@ -806,927 +920,6 @@ snmp_view_basec_acm_mib:vacmAccessTable(set, RowIndex, Cols). </section> <!-- 4.15 --> - <section> - <title>SNMP Development Toolkit 4.14</title> - - <p>Version 4.14 supports code replacement in runtime from/to - version 4.13.5, 4.13.4, 4.13.3, 4.13.2, 4.13.1 and 4.13.</p> - - <section> - <title>Improvements and new features</title> - <!-- - <p>-</p> - --> - - <list type="bulleted"> - <item> - <p>[compiler] Include object- and notification groups in the - compiled mib. - This will make it possible to import groups from other mibs. </p> - <p>Also the SNMPv2-MIB-file has been updated to a more - up-to-date version. </p> - <p>Own Id: OTP-8223</p> - <!-- <p>Aux Id: Seq 11383</p> --> - </item> - - <item> - <p>[manager] Added support for message filtering in the - network interface module provided with the application. - The component that actually make the filter decisions - is the network interface filter module. This module - must implement the - <seealso marker="snmpm_network_interface_filter">network interface filter behaviour</seealso> - for message filtering. - See also the Configuring chapter of - the User's Guide to see how to configure this feature. </p> - <p>See the - <seealso marker="snmp_app#configuration_params">configuration</seealso> - chapter for more info about the filter options.</p> - <p>Own Id: OTP-8228</p> - <p>Aux Id: Seq 11411</p> - </item> - - <item> - <p>The MIBs delivered as part of the application is now - also available as man pages, section 7. </p> - <p>Own Id: OTP-8237</p> - <!-- <p>Aux Id: Seq 11383</p> --> - </item> - - </list> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <p>-</p> - - <!-- - <list type="bulleted"> - <item> - <p>[agent] The main agent type header file contained some miss-information - regarding the type of the entrytype field of the me-record, causing - unneccessary confusion.</p> - <p>Own Id: OTP-8116</p> - <p>Aux Id: Seq 11312</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.14 --> - - - <section> - <title>SNMP Development Toolkit 4.13.5</title> - - <p>Version 4.13.5 supports code replacement in runtime from/to - version 4.13.4, 4.13.3, 4.13.2, 4.13.1 and 4.13.</p> - - <section> - <title>Improvements and new features</title> - <!-- - <p>-</p> - --> - - <list type="bulleted"> - <item> - <p>[agent] Improved the cache handling of the mib server. </p> - <p>A number of new functions and config options for the mib server - cache has been added. </p> - <p>See - <seealso marker="snmpa#invalidate_mibs_cache">invalidate_mibs_cache/0,1</seealso>, - <seealso marker="snmpa#enable_mibs_cache">enable_mibs_cache/0,1</seealso>, - <seealso marker="snmpa#disable_mibs_cache">disable_mibs_cache/0,1</seealso>, - <seealso marker="snmpa#gc_mibs_cache">gc_mibs_cache/0,1,2,3</seealso>, - <seealso marker="snmpa#enable_mibs_cache_autogc">enable_mibs_cache_autogc/0,1</seealso>, - <seealso marker="snmpa#disable_mibs_cache_autogc">disable_mibs_cache_autogc/0,1</seealso>, - <seealso marker="snmpa#update_mibs_cache_age">update_mibs_cache_age/1,2</seealso> and - <seealso marker="snmpa#update_mibs_cache_gclimit">update_mibs_cache_gclimit/1,2</seealso> for more info. </p> - <p>See also the - <seealso marker="snmp_app#configuration_params">configuration</seealso> - chapter for more info about the mib server cache options.</p> - <p>Own Id: OTP-8182</p> - <p>Aux Id: Seq 11383</p> - </item> - - <item> - <p>[agent] A manager could no longer use the SNMPv3 user "initial" - as this was interpretated as the first step of the discovery. </p> - <p>Introduced a new terminating option, <c>trigger_username</c> to - make it possible to configure the username the agent reacts to. - Default is <c>""</c>. </p> - <p>See the - <seealso marker="snmp_app#configuration_params">configuration</seealso> - chapter for more info about the discovery options.</p> - <p>Own Id: OTP-8120</p> - <p>Aux Id: Seq 11361</p> - </item> - - </list> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[agent] The main agent type header file contained some miss-information - regarding the type of the entrytype field of the me-record, causing - unneccessary confusion.</p> - <p>Own Id: OTP-8116</p> - <p>Aux Id: Seq 11312</p> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.13.5 --> - - - <section> - <title>SNMP Development Toolkit 4.13.4</title> - - <p>Version 4.13.4 supports code replacement in runtime from/to - version 4.13.3, 4.13.2, 4.13.1 and 4.13.</p> - - <section> - <title>Improvements and new features</title> - <p>-</p> - - <!-- - <list type="bulleted"> - <item> - <p>[agent] Support for the discovery process. </p> - <p>The agent can both initiate discovery itself (see the - <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter - for more info) and respond to discovery initiated by a manager.</p> - <p>Own Id: OTP-7571</p> - <p>Aux Id: Seq 11053</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[agent] Originating discovery problems. </p> - <p>Invalid state variable update during second stage of - discovery causes master agent crash. </p> - <p>Also the net_if process failed to activate socket - ({active, once}) after first discovery response was sent. </p> - <p>Own Id: OTP-8044</p> - <p>Aux Id: Seq 11295</p> - </item> - - <item> - <p>[agent] Terminating discovery problem. </p> - <p>The reply to the second stage request should include a - varbind with <c>usmStatsNotInTimeWindows</c>.</p> - <p>Own Id: OTP-8062</p> - <p>Aux Id: Seq 11318</p> - </item> - - <item> - <p>[agent] Originating discovery improvement. </p> - <p>Added the ExtraInfo argument to the - <seealso marker="snmpa#discovery">discovery</seealso> function. - This argument will be passed on to the stage1_finish callback - function. Also, the - <seealso marker="snmpa#discovery">discovery</seealso> function - will now always return <c>{ok, ManagerEngineID}</c> on successful - discovery. </p> - <p>The <seealso marker="snmpa_discovery_handler">discovery handler</seealso> - behaviour updated accordingly. </p> - <p>Own Id: OTP-8098</p> - <p>Aux Id: Seq 11346</p> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.13.4 --> - - - <section> - <title>SNMP Development Toolkit 4.13.3</title> - - <p>Version 4.13.3 supports code replacement in runtime from/to - version 4.13.2, 4.13.1 and 4.13.</p> - - <section> - <title>Improvements and new features</title> - <p>-</p> - - <!-- - <list type="bulleted"> - <item> - <p>[agent] Support for the discovery process. </p> - <p>The agent can both initiate discovery itself (see the - <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter - for more info) and respond to discovery initiated by a manager.</p> - <p>Own Id: OTP-7571</p> - <p>Aux Id: Seq 11053</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[manager] A request for an oid of type BITS was actually - returned as OCTET STRING. </p> - <p>Values of type BITS are encoded as OCTET STRING, - which makes it impossible for the decoder to know that - they should really be of type BITS. - Instead, this has to be done higher up in the stack, where - there is knowledge of the MIB (assuming that the mib has - been loaded, there is info about the type of the mibentry). </p> - <p>This problem has now been fixed, but requires that the MIB - defining this mib-entry is loaded! </p> - <p>The utility function - <seealso marker="snmpm#oid_to_type">oid_to_type</seealso> - has been added, for debug purpose. </p> - <p>The utility function(s) - <seealso marker="snmp#octet_string_to_bits">octet_string_to_bits</seealso> - and - <seealso marker="snmp#bits_to_octet_string">bits_to_octet_string</seealso> - has also been added. These can be used if the user prefers to - handle the conversion on their own. </p> - <p>Own Id: OTP-8015</p> - <p>Aux Id: Seq 11285</p> - </item> - - <item> - <p>[agent] Fixed some issues with the discovery handling. </p> - <p>Changed the API of the - <seealso marker="snmpa#discovery">discovery</seealso> - function to solve some - of these problems. </p> - <p>Introduced various options for controlling the discovery - process. See the - <seealso marker="snmp_app#configuration_params">configuration</seealso> - chapter for more info about the discovery options.</p> - <p>Own Id: OTP-8020</p> - <p>Aux Id: Seq 11295</p> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.13.3 --> - - - <section> - <title>SNMP Development Toolkit 4.13.2</title> - - <p>Version 4.13.2 supports code replacement in runtime from/to - version 4.13.1 and 4.13.</p> - - <section> - <title>Improvements and new features</title> - <p>-</p> - - <!-- - <list type="bulleted"> - <item> - <p>[agent] Support for the discovery process. </p> - <p>The agent can both initiate discovery itself (see the - <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter - for more info) and respond to discovery initiated by a manager.</p> - <p>Own Id: OTP-7571</p> - <p>Aux Id: Seq 11053</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[manager] Failure during downed user cleanup. - As part of the cleanup after a crashed user, - the manager attempts to unregister the agents - registered by this user. This however failed, - causing a server crash. </p> - <p>Own Id: OTP-7961</p> - <p>Aux Id: Seq 11275</p> - </item> - - <item> - <p>[manager] Incorrectly documented value type for - IpAddress (ip). The value type for IpAddress is - documented as ip but is actually ia. The value type - ip has been added. The old (not documented) value - type ia still works. </p> - <p>Own Id: OTP-7977</p> - <p>Aux Id: Seq 11279</p> - </item> - - <item> - <p>[manager] EngineId lookup fails when using version-3. </p> - <p>Own Id: OTP-7983</p> - <p>Aux Id: Seq 11275</p> - </item> - - <item> - <p>[agent] As of version 4.13 the possible return values - of the function - <seealso marker="snmpa_mpd#process_packet">snmpa_mpd:process_packet/4</seealso> - changed, but this was not documented. </p> - <p>Own Id: OTP-7989</p> - <p>Aux Id: Seq 11275</p> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.13.2 --> - - <section> - <title>SNMP Development Toolkit 4.13.1</title> - - <p>Version 4.13.1 supports code replacement in runtime from/to - version 4.13.</p> - - <section> - <title>Improvements and new features</title> - <p>-</p> - - <!-- - <list type="bulleted"> - <item> - <p>[agent] Support for the discovery process. </p> - <p>The agent can both initiate discovery itself (see the - <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter - for more info) and respond to discovery initiated by a manager.</p> - <p>Own Id: OTP-7571</p> - <p>Aux Id: Seq 11053</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[manager] Registration of users had some issues. </p> - <p>Not all of the registration functions where actually exported - (<seealso marker="snmpm#register_user">register_user/4</seealso> - and - <seealso marker="snmpm#register_user_monitor">register_user_monitor/4</seealso>). - This has now been fixed. </p> - <p>Also, the registration did not succeed unless - user implemented the *new* behaviour. This has now - also been fixed (registration succeeds if the user - implements either the new (i.e. updated - <seealso marker="snmpm_user">snmpm_user</seealso>) - or the old user behaviour (<c>snmpm_user_old</c>)). </p> - <p>Own Id: OTP-7902</p> - <p>Aux Id: Seq 11240</p> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.13.1 --> - - <section> - <title>SNMP Development Toolkit 4.13</title> -<!-- - <p>Version 4.13 supports code replacement in runtime from/to - version 4.12.1.</p> ---> - - <section> - <title>Improvements and new features</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[agent] Support for the discovery process. </p> - <p>The agent can both initiate discovery itself (see the - <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter - for more info) and respond to discovery initiated by a manager.</p> - <p>Own Id: OTP-7571</p> - <p>Aux Id: Seq 11053</p> - </item> - - </list> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[agent] Unnecessary use of math:pow/2 could cause problems - on systems without floating point support. </p> - <p>Per Hedeland</p> - <p>Own Id: OTP-7735</p> - <!-- <p>Aux Id: Seq 10966</p> --> - </item> - - <item> - <p>[manager] A major flaw was discovered with the agent handling. </p> - <p>First, <c>TargetName</c> was never used as intended, as a unique - identifier for the target (agent in this case). </p> - <p>Second, <c>TargetName</c> had a <em>default value</em>, which meant - that several agents could have the same <c>TargetName</c>, causing - unpredictable behaviour in the manager. </p> - <p>Third, <c>EngineID</c> was not a mandatory config option and had - furthermore also a <em>default value</em>. </p> - - <p>These problems has been solved in the following way: </p> - <p>First, a new set of api functions has been introduced (and documented): - <seealso marker="snmpm#register_user">register_user/4</seealso>, - <seealso marker="snmpm#register_user_monitor">register_user_monitor/4</seealso>, - <seealso marker="snmpm#register_agent">register_agent/3</seealso>, - <seealso marker="snmpm#unregister_agent">unregister_agent/2</seealso>, - <seealso marker="snmpm#agent_info">agent_info/2</seealso>, - <seealso marker="snmpm#update_agent_info">update_agent_info/4</seealso>, - <seealso marker="snmpm#sync_get">sync_get/3,4,5,6</seealso>, - <seealso marker="snmpm#async_get">async_get/3,4,5,6</seealso>, - <seealso marker="snmpm#sync_get_next">sync_get_next/3,4,5,6</seealso>, - <seealso marker="snmpm#async_get_next">async_get_next/3,4,5,6</seealso>, - <seealso marker="snmpm#sync_set">sync_set/3,4,5,6</seealso>, - <seealso marker="snmpm#async_set">async_set/3,4,5,6</seealso>, - <seealso marker="snmpm#sync_get_bulk">sync_get_bulk/5,6,7,8</seealso> and - <seealso marker="snmpm#async_get_bulk">async_get_bulk/5,6,7,8</seealso> - that all use <c>TargetName</c> (and not, as previously, <c>Addr</c> - and <c>Port</c>) to identify the agent (also the return value of - <seealso marker="snmpm#which_agents">which_agents</seealso> has - been changed). </p> - <p>Second, for backward compatibility, the old functions still - exist, but are no longer documented and are now wrappers for the - new functions, including erroneous default value for EngineID and - all. The TargetName is however generated from the provided - <c>Addr</c>, <c>Port</c> and <c>Version</c> config options. </p> - <p>Third, the behaviour of the - <seealso marker="snmpm_user">SNMP manager user</seealso> has - been changed to reflect this, i.e. - <seealso marker="snmpm_user#handle_pdu">handle_pdu/4</seealso>, - <seealso marker="snmpm_user#handle_trap">handle_trap/3</seealso>, - <seealso marker="snmpm_user#handle_inform">handle_inform/3</seealso>, - <seealso marker="snmpm_user#handle_report">handle_report/3</seealso> - and the return-value of - <seealso marker="snmpm_user#handle_agent">handle_agent/4</seealso>. - The old (non-documented) callback-functions (using Addr and Port) - will still be called if the agent was registered using the old - registration functions. </p> - - <p>Own Id: OTP-7836</p> - <!-- <p>Aux Id: Seq 10966</p> --> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.13 --> - - <section> - <title>SNMP Development Toolkit 4.12.2</title> - <p>Version 4.12.2 supports code replacement in runtime from/to - version 4.12.1, 4.12, 4.11.2, 4.11.1 and 4.11.</p> - - <section> - <title>Improvements and new features</title> - <p>-</p> - <!-- - <list type="bulleted"> - <item> - <item> - <p>[agent] Improvement of the inform reporting. - It was previously not certain how many acks an - application received, 0, 1 or 2. This has now been - fixed, so that only 1 (one) ack is issued. </p> - <p>Per Hedeland</p> - <p>Own Id: OTP-7525</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[agent] Bad session cache (usm+camv-info) invalidation - could cause user crash, through call(s) to (a number of) - MIB API function(s) (undefined function). </p> - <p>Own Id: OTP-7868</p> - <!-- <p>Aux Id: Seq 11124</p> --> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.12.2 --> - - <section> - <title>SNMP Development Toolkit 4.12.1</title> - <p>Version 4.12.1 supports code replacement in runtime from/to - version 4.12, 4.11.2, 4.11.1 and 4.11.</p> - - <section> - <title>Improvements and new features</title> - <p>-</p> - <!-- - <list type="bulleted"> - <item> - <item> - <p>[agent] Improvement of the inform reporting. - It was previously not certain how many acks an - application received, 0, 1 or 2. This has now been - fixed, so that only 1 (one) ack is issued. </p> - <p>Per Hedeland</p> - <p>Own Id: OTP-7525</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>Logging of messages with the GetBulk-request PDU - incorrectly produced an erroneous entry in the - log: "An error occurred". </p> - <p>The reason for this was that the PDU-fields - error_status and error_index is re-used for - Non-repeaters and Max-repetitions for - GetBulk-request PDUs, but this was not handled - by the logging code. </p> - <p>Own Id: OTP-7695</p> - <p>Aux Id: Seq 11124</p> - </item> - - <item> - <p>[agent] An attempt to set the row status to active for an - notReady table row, could result in an "inconsistentValue" - error. </p> - <p>The same problem existed when attempting to set row status - to notInService for a row in notReady. </p> - <p>Serge Aleynikov</p> - <p>Own Id: OTP-7698</p> - <!-- <p>Aux Id: Seq 10966</p> --> - </item> - - </list> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.12.1 --> - - <section> - <title>SNMP Development Toolkit 4.12</title> - <p>Version 4.12 supports code replacement in runtime from/to - version 4.11.2, 4.11.1 and 4.11.</p> - - <section> - <title>Improvements and new features</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[agent] A simple lookup cache has been added to improve - the mib server lookup performance. </p> - <p>This can be disabled with the mib_server - <seealso marker="snmp_app">cache</seealso> option. </p> - <p>Own Id: OTP-7346</p> - </item> - - <item> - <p>[agent] Improvement of the inform reporting. - It was previously not certain how many acks an - application received, 0, 1 or 2. This has now been - fixed, so that only 1 (one) ack is issued. </p> - <p>Per Hedeland</p> - <p>Own Id: OTP-7525</p> - </item> - - </list> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <p>-</p> - <!-- - <list type="bulleted"> - <item> - <p>[manager] Encryption error when attempting to send - version 3 inform-requests. </p> - <p>Own Id: OTP-7432</p> - <p>Aux Id: Seq 10966</p> - </item> - - </list> - --> - - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.12 --> - - <section> - <title>SNMP Development Toolkit 4.11.2</title> - <p>Version 4.11.2 supports code replacement in runtime from/to - version 4.11.1 and 4.11. </p> - - <section> - <title>Improvements and new features</title> - <p>-</p> - <!-- - <list type="bulleted"> - <item> - <p>Added utility functions for transforming DateAndTime - as [int()] to strings; - <seealso marker="snmp#dat2s">date_and_time_to_string/2</seealso> - and - <seealso marker="snmp#dat2s2">date_and_time_to_string2/1</seealso>. </p> - <p>Also added new validation function - <seealso marker="snmp#vdat">validate_date_and_time/2</seealso>. </p> - <p>Own Id: OTP-7412</p> - <p>Aux Id: Seq 10987</p> - </item> - </list> - --> - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[manager] Erroneous engine-id check when receiving version 3 - informs. </p> - <p>Own Id: OTP-7570</p> - <p>Aux Id: Seq 11060</p> - </item> - - <item> - <p>Receiving an snmp message with a very large version - number could cause the erlang node to run out of - memory and consequently crash. </p> - <p>The standard specifies the snmp version as an - (unlimited) INTEGER, but today only - 0 (version 1), 1 (version 2) and 3 (version 3) is - actually used. So, when decoding a message, a limit - has been put on the snmp version integer in order - to not allow this kind of a problem. </p> - <p>Own Id: OTP-7575</p> - <p>Aux Id: Seq 11064</p> - </item> - - </list> - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.11.2 --> - - - <section> - <title>SNMP Development Toolkit 4.11.1</title> - <p>Version 4.11.1 supports code replacement in runtime from/to - version 4.11.</p> - - <section> - <title>Improvements and new features</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[compiler] The MIB compiler did not retrieve the REFERENCE part - of a SNMP MIB definition. </p> - <p>This problem has been partly solved. For SNMP tables, - the assocList field of the tables mib-entry record now contains - this info (as <c>{reference, string()}</c>), <em>if</em> the - MIB was compiled with the compiler option <em>+reference</em>. </p> - <p>This solution is temporary, until such time as a permanent - solution (and probably not backward compatible) is devised, which - retrieves and stores all REFERENCE part(s) of a MIB. </p> - <p>See the - <seealso marker="snmpc#compiler_opts">compiler options</seealso> - for more info. </p> - - <p>Serge Aleynikov</p> - <p>Own Id: OTP-7426</p> - </item> - - <item> - <p>Added utility functions for transforming DateAndTime - as [int()] to strings; - <seealso marker="snmp#dat2s">date_and_time_to_string/2</seealso> - and - <seealso marker="snmp#dat2s2">date_and_time_to_string2/1</seealso>. </p> - <p>Also added new validation function - <seealso marker="snmp#vdat">validate_date_and_time/2</seealso>. </p> - <p>Own Id: OTP-7412</p> - <p>Aux Id: Seq 10987</p> - </item> - - </list> - - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[manager] Encryption error when attempting to send - version 3 inform-requests. </p> - <p>Own Id: OTP-7432</p> - <p>Aux Id: Seq 10966</p> - </item> - - </list> - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.11.1 --> - - <section> - <title>SNMP Development Toolkit 4.11</title> - <p>Version 4.11 supports code replacement in runtime from/to - version 4.10.3, 4.10.2, 4.10.1 and 4.10.</p> - - <section> - <title>Improvements and new features</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[agent] Performance improvements in the case when an SNMP - manager performs an snmpwalk. </p> - <p>Martin Björklund</p> - <p>Own Id: OTP-7201</p> - </item> - - <item> - <p>The API for sending inform(s) has been improved. Also - the documentation has been corrected and updated. See - <seealso marker="snmpa#send_notification">snmpa:send_notification</seealso> and - <seealso marker="snmpa_notification_delivery_info_receiver">snmpa_notification_delivery_info_receiver</seealso> - for more info.</p> - <p>Own Id: OTP-7287</p> - <p>Aux Id: Seq 10926</p> - </item> - - <item> - <p>[agent] Performance of the internal database (local-db) - has been improved.</p> - <p>Own Id: OTP-7319</p> - <p>Aux Id: Seq 10942</p> - </item> - - <item> - <p>[agent] Added utility functions, - <seealso marker="snmpa#restart_worker">snmpa:restart_worker/0,1</seealso> and - <seealso marker="snmpa#restart_set_worker">snmpa:restart_set_worker/0,1</seealso>, - for restarting the agent worker processes (in case the agent is - multi-threaded).</p> - <p>Own Id: OTP-7369</p> - </item> - - <item> - <p>Add utility function to - <seealso marker="snmp#read_mib">read</seealso> - a compiled mib. </p> - <p>Own Id: OTP-7371</p> - </item> - - </list> - </section> - - <section> - <title>Reported Fixed Bugs and Malfunctions</title> - <!-- - <p>-</p> - --> - <list type="bulleted"> - <item> - <p>[manager] Encryption error when attempting to send - version 3 inform-requests. </p> - <p>Own Id: OTP-7377</p> - <p>Aux Id: Seq 10966</p> - </item> - - </list> - </section> - - <section> - <title>Incompatibilities</title> - <p>-</p> - </section> - </section> <!-- 4.11 --> - <!-- section> <title>Release notes history</title> <p>For information about older versions see diff --git a/lib/snmp/doc/src/notes_history.xml b/lib/snmp/doc/src/notes_history.xml index 934df87866..722d02afbd 100644 --- a/lib/snmp/doc/src/notes_history.xml +++ b/lib/snmp/doc/src/notes_history.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="iso-8859-1" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> @@ -33,6 +33,928 @@ </header> <section> + <title>SNMP Development Toolkit 4.14</title> + + <p>Version 4.14 supports code replacement in runtime from/to + version 4.13.5, 4.13.4, 4.13.3, 4.13.2, 4.13.1 and 4.13.</p> + + <section> + <title>Improvements and new features</title> + <!-- + <p>-</p> + --> + + <list type="bulleted"> + <item> + <p>[compiler] Include object- and notification groups in the + compiled mib. + This will make it possible to import groups from other mibs. </p> + <p>Also the SNMPv2-MIB-file has been updated to a more + up-to-date version. </p> + <p>Own Id: OTP-8223</p> + <!-- <p>Aux Id: Seq 11383</p> --> + </item> + + <item> + <p>[manager] Added support for message filtering in the + network interface module provided with the application. + The component that actually make the filter decisions + is the network interface filter module. This module + must implement the + <seealso marker="snmpm_network_interface_filter">network interface filter behaviour</seealso> + for message filtering. + See also the Configuring chapter of + the User's Guide to see how to configure this feature. </p> + <p>See the + <seealso marker="snmp_app#configuration_params">configuration</seealso> + chapter for more info about the filter options.</p> + <p>Own Id: OTP-8228</p> + <p>Aux Id: Seq 11411</p> + </item> + + <item> + <p>The MIBs delivered as part of the application is now + also available as man pages, section 7. </p> + <p>Own Id: OTP-8237</p> + <!-- <p>Aux Id: Seq 11383</p> --> + </item> + + </list> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <p>-</p> + + <!-- + <list type="bulleted"> + <item> + <p>[agent] The main agent type header file contained some miss-information + regarding the type of the entrytype field of the me-record, causing + unneccessary confusion.</p> + <p>Own Id: OTP-8116</p> + <p>Aux Id: Seq 11312</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.14 --> + + + <section> + <title>SNMP Development Toolkit 4.13.5</title> + + <p>Version 4.13.5 supports code replacement in runtime from/to + version 4.13.4, 4.13.3, 4.13.2, 4.13.1 and 4.13.</p> + + <section> + <title>Improvements and new features</title> + <!-- + <p>-</p> + --> + + <list type="bulleted"> + <item> + <p>[agent] Improved the cache handling of the mib server. </p> + <p>A number of new functions and config options for the mib server + cache has been added. </p> + <p>See + <seealso marker="snmpa#invalidate_mibs_cache">invalidate_mibs_cache/0,1</seealso>, + <seealso marker="snmpa#enable_mibs_cache">enable_mibs_cache/0,1</seealso>, + <seealso marker="snmpa#disable_mibs_cache">disable_mibs_cache/0,1</seealso>, + <seealso marker="snmpa#gc_mibs_cache">gc_mibs_cache/0,1,2,3</seealso>, + <seealso marker="snmpa#enable_mibs_cache_autogc">enable_mibs_cache_autogc/0,1</seealso>, + <seealso marker="snmpa#disable_mibs_cache_autogc">disable_mibs_cache_autogc/0,1</seealso>, + <seealso marker="snmpa#update_mibs_cache_age">update_mibs_cache_age/1,2</seealso> and + <seealso marker="snmpa#update_mibs_cache_gclimit">update_mibs_cache_gclimit/1,2</seealso> for more info. </p> + <p>See also the + <seealso marker="snmp_app#configuration_params">configuration</seealso> + chapter for more info about the mib server cache options.</p> + <p>Own Id: OTP-8182</p> + <p>Aux Id: Seq 11383</p> + </item> + + <item> + <p>[agent] A manager could no longer use the SNMPv3 user "initial" + as this was interpretated as the first step of the discovery. </p> + <p>Introduced a new terminating option, <c>trigger_username</c> to + make it possible to configure the username the agent reacts to. + Default is <c>""</c>. </p> + <p>See the + <seealso marker="snmp_app#configuration_params">configuration</seealso> + chapter for more info about the discovery options.</p> + <p>Own Id: OTP-8120</p> + <p>Aux Id: Seq 11361</p> + </item> + + </list> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[agent] The main agent type header file contained some miss-information + regarding the type of the entrytype field of the me-record, causing + unneccessary confusion.</p> + <p>Own Id: OTP-8116</p> + <p>Aux Id: Seq 11312</p> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.13.5 --> + + + <section> + <title>SNMP Development Toolkit 4.13.4</title> + + <p>Version 4.13.4 supports code replacement in runtime from/to + version 4.13.3, 4.13.2, 4.13.1 and 4.13.</p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + + <!-- + <list type="bulleted"> + <item> + <p>[agent] Support for the discovery process. </p> + <p>The agent can both initiate discovery itself (see the + <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter + for more info) and respond to discovery initiated by a manager.</p> + <p>Own Id: OTP-7571</p> + <p>Aux Id: Seq 11053</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[agent] Originating discovery problems. </p> + <p>Invalid state variable update during second stage of + discovery causes master agent crash. </p> + <p>Also the net_if process failed to activate socket + ({active, once}) after first discovery response was sent. </p> + <p>Own Id: OTP-8044</p> + <p>Aux Id: Seq 11295</p> + </item> + + <item> + <p>[agent] Terminating discovery problem. </p> + <p>The reply to the second stage request should include a + varbind with <c>usmStatsNotInTimeWindows</c>.</p> + <p>Own Id: OTP-8062</p> + <p>Aux Id: Seq 11318</p> + </item> + + <item> + <p>[agent] Originating discovery improvement. </p> + <p>Added the ExtraInfo argument to the + <seealso marker="snmpa#discovery">discovery</seealso> function. + This argument will be passed on to the stage1_finish callback + function. Also, the + <seealso marker="snmpa#discovery">discovery</seealso> function + will now always return <c>{ok, ManagerEngineID}</c> on successful + discovery. </p> + <p>The <seealso marker="snmpa_discovery_handler">discovery handler</seealso> + behaviour updated accordingly. </p> + <p>Own Id: OTP-8098</p> + <p>Aux Id: Seq 11346</p> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.13.4 --> + + + <section> + <title>SNMP Development Toolkit 4.13.3</title> + + <p>Version 4.13.3 supports code replacement in runtime from/to + version 4.13.2, 4.13.1 and 4.13.</p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + + <!-- + <list type="bulleted"> + <item> + <p>[agent] Support for the discovery process. </p> + <p>The agent can both initiate discovery itself (see the + <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter + for more info) and respond to discovery initiated by a manager.</p> + <p>Own Id: OTP-7571</p> + <p>Aux Id: Seq 11053</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[manager] A request for an oid of type BITS was actually + returned as OCTET STRING. </p> + <p>Values of type BITS are encoded as OCTET STRING, + which makes it impossible for the decoder to know that + they should really be of type BITS. + Instead, this has to be done higher up in the stack, where + there is knowledge of the MIB (assuming that the mib has + been loaded, there is info about the type of the mibentry). </p> + <p>This problem has now been fixed, but requires that the MIB + defining this mib-entry is loaded! </p> + <p>The utility function + <seealso marker="snmpm#oid_to_type">oid_to_type</seealso> + has been added, for debug purpose. </p> + <p>The utility function(s) + <seealso marker="snmp#octet_string_to_bits">octet_string_to_bits</seealso> + and + <seealso marker="snmp#bits_to_octet_string">bits_to_octet_string</seealso> + has also been added. These can be used if the user prefers to + handle the conversion on their own. </p> + <p>Own Id: OTP-8015</p> + <p>Aux Id: Seq 11285</p> + </item> + + <item> + <p>[agent] Fixed some issues with the discovery handling. </p> + <p>Changed the API of the + <seealso marker="snmpa#discovery">discovery</seealso> + function to solve some + of these problems. </p> + <p>Introduced various options for controlling the discovery + process. See the + <seealso marker="snmp_app#configuration_params">configuration</seealso> + chapter for more info about the discovery options.</p> + <p>Own Id: OTP-8020</p> + <p>Aux Id: Seq 11295</p> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.13.3 --> + + + <section> + <title>SNMP Development Toolkit 4.13.2</title> + + <p>Version 4.13.2 supports code replacement in runtime from/to + version 4.13.1 and 4.13.</p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + + <!-- + <list type="bulleted"> + <item> + <p>[agent] Support for the discovery process. </p> + <p>The agent can both initiate discovery itself (see the + <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter + for more info) and respond to discovery initiated by a manager.</p> + <p>Own Id: OTP-7571</p> + <p>Aux Id: Seq 11053</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[manager] Failure during downed user cleanup. + As part of the cleanup after a crashed user, + the manager attempts to unregister the agents + registered by this user. This however failed, + causing a server crash. </p> + <p>Own Id: OTP-7961</p> + <p>Aux Id: Seq 11275</p> + </item> + + <item> + <p>[manager] Incorrectly documented value type for + IpAddress (ip). The value type for IpAddress is + documented as ip but is actually ia. The value type + ip has been added. The old (not documented) value + type ia still works. </p> + <p>Own Id: OTP-7977</p> + <p>Aux Id: Seq 11279</p> + </item> + + <item> + <p>[manager] EngineId lookup fails when using version-3. </p> + <p>Own Id: OTP-7983</p> + <p>Aux Id: Seq 11275</p> + </item> + + <item> + <p>[agent] As of version 4.13 the possible return values + of the function + <seealso marker="snmpa_mpd#process_packet">snmpa_mpd:process_packet/4</seealso> + changed, but this was not documented. </p> + <p>Own Id: OTP-7989</p> + <p>Aux Id: Seq 11275</p> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.13.2 --> + + <section> + <title>SNMP Development Toolkit 4.13.1</title> + + <p>Version 4.13.1 supports code replacement in runtime from/to + version 4.13.</p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + + <!-- + <list type="bulleted"> + <item> + <p>[agent] Support for the discovery process. </p> + <p>The agent can both initiate discovery itself (see the + <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter + for more info) and respond to discovery initiated by a manager.</p> + <p>Own Id: OTP-7571</p> + <p>Aux Id: Seq 11053</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[manager] Registration of users had some issues. </p> + <p>Not all of the registration functions where actually exported + (<seealso marker="snmpm#register_user">register_user/4</seealso> + and + <seealso marker="snmpm#register_user_monitor">register_user_monitor/4</seealso>). + This has now been fixed. </p> + <p>Also, the registration did not succeed unless + user implemented the *new* behaviour. This has now + also been fixed (registration succeeds if the user + implements either the new (i.e. updated + <seealso marker="snmpm_user">snmpm_user</seealso>) + or the old user behaviour (<c>snmpm_user_old</c>)). </p> + <p>Own Id: OTP-7902</p> + <p>Aux Id: Seq 11240</p> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.13.1 --> + + <section> + <title>SNMP Development Toolkit 4.13</title> +<!-- + <p>Version 4.13 supports code replacement in runtime from/to + version 4.12.1.</p> +--> + + <section> + <title>Improvements and new features</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[agent] Support for the discovery process. </p> + <p>The agent can both initiate discovery itself (see the + <seealso marker="snmp_agent_funct_descr#discovery">discovery</seealso> chapter + for more info) and respond to discovery initiated by a manager.</p> + <p>Own Id: OTP-7571</p> + <p>Aux Id: Seq 11053</p> + </item> + + </list> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[agent] Unnecessary use of math:pow/2 could cause problems + on systems without floating point support. </p> + <p>Per Hedeland</p> + <p>Own Id: OTP-7735</p> + <!-- <p>Aux Id: Seq 10966</p> --> + </item> + + <item> + <p>[manager] A major flaw was discovered with the agent handling. </p> + <p>First, <c>TargetName</c> was never used as intended, as a unique + identifier for the target (agent in this case). </p> + <p>Second, <c>TargetName</c> had a <em>default value</em>, which meant + that several agents could have the same <c>TargetName</c>, causing + unpredictable behaviour in the manager. </p> + <p>Third, <c>EngineID</c> was not a mandatory config option and had + furthermore also a <em>default value</em>. </p> + + <p>These problems has been solved in the following way: </p> + <p>First, a new set of api functions has been introduced (and documented): + <seealso marker="snmpm#register_user">register_user/4</seealso>, + <seealso marker="snmpm#register_user_monitor">register_user_monitor/4</seealso>, + <seealso marker="snmpm#register_agent">register_agent/3</seealso>, + <seealso marker="snmpm#unregister_agent">unregister_agent/2</seealso>, + <seealso marker="snmpm#agent_info">agent_info/2</seealso>, + <seealso marker="snmpm#update_agent_info">update_agent_info/4</seealso>, + <seealso marker="snmpm#sync_get">sync_get/3,4,5,6</seealso>, + <seealso marker="snmpm#async_get">async_get/3,4,5,6</seealso>, + <seealso marker="snmpm#sync_get_next">sync_get_next/3,4,5,6</seealso>, + <seealso marker="snmpm#async_get_next">async_get_next/3,4,5,6</seealso>, + <seealso marker="snmpm#sync_set">sync_set/3,4,5,6</seealso>, + <seealso marker="snmpm#async_set">async_set/3,4,5,6</seealso>, + <seealso marker="snmpm#sync_get_bulk">sync_get_bulk/5,6,7,8</seealso> and + <seealso marker="snmpm#async_get_bulk">async_get_bulk/5,6,7,8</seealso> + that all use <c>TargetName</c> (and not, as previously, <c>Addr</c> + and <c>Port</c>) to identify the agent (also the return value of + <seealso marker="snmpm#which_agents">which_agents</seealso> has + been changed). </p> + <p>Second, for backward compatibility, the old functions still + exist, but are no longer documented and are now wrappers for the + new functions, including erroneous default value for EngineID and + all. The TargetName is however generated from the provided + <c>Addr</c>, <c>Port</c> and <c>Version</c> config options. </p> + <p>Third, the behaviour of the + <seealso marker="snmpm_user">SNMP manager user</seealso> has + been changed to reflect this, i.e. + <seealso marker="snmpm_user#handle_pdu">handle_pdu/4</seealso>, + <seealso marker="snmpm_user#handle_trap">handle_trap/3</seealso>, + <seealso marker="snmpm_user#handle_inform">handle_inform/3</seealso>, + <seealso marker="snmpm_user#handle_report">handle_report/3</seealso> + and the return-value of + <seealso marker="snmpm_user#handle_agent">handle_agent/4</seealso>. + The old (non-documented) callback-functions (using Addr and Port) + will still be called if the agent was registered using the old + registration functions. </p> + + <p>Own Id: OTP-7836</p> + <!-- <p>Aux Id: Seq 10966</p> --> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.13 --> + + <section> + <title>SNMP Development Toolkit 4.12.2</title> + <p>Version 4.12.2 supports code replacement in runtime from/to + version 4.12.1, 4.12, 4.11.2, 4.11.1 and 4.11.</p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + <!-- + <list type="bulleted"> + <item> + <item> + <p>[agent] Improvement of the inform reporting. + It was previously not certain how many acks an + application received, 0, 1 or 2. This has now been + fixed, so that only 1 (one) ack is issued. </p> + <p>Per Hedeland</p> + <p>Own Id: OTP-7525</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[agent] Bad session cache (usm+camv-info) invalidation + could cause user crash, through call(s) to (a number of) + MIB API function(s) (undefined function). </p> + <p>Own Id: OTP-7868</p> + <!-- <p>Aux Id: Seq 11124</p> --> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.12.2 --> + + <section> + <title>SNMP Development Toolkit 4.12.1</title> + <p>Version 4.12.1 supports code replacement in runtime from/to + version 4.12, 4.11.2, 4.11.1 and 4.11.</p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + <!-- + <list type="bulleted"> + <item> + <item> + <p>[agent] Improvement of the inform reporting. + It was previously not certain how many acks an + application received, 0, 1 or 2. This has now been + fixed, so that only 1 (one) ack is issued. </p> + <p>Per Hedeland</p> + <p>Own Id: OTP-7525</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>Logging of messages with the GetBulk-request PDU + incorrectly produced an erroneous entry in the + log: "An error occurred". </p> + <p>The reason for this was that the PDU-fields + error_status and error_index is re-used for + Non-repeaters and Max-repetitions for + GetBulk-request PDUs, but this was not handled + by the logging code. </p> + <p>Own Id: OTP-7695</p> + <p>Aux Id: Seq 11124</p> + </item> + + <item> + <p>[agent] An attempt to set the row status to active for an + notReady table row, could result in an "inconsistentValue" + error. </p> + <p>The same problem existed when attempting to set row status + to notInService for a row in notReady. </p> + <p>Serge Aleynikov</p> + <p>Own Id: OTP-7698</p> + <!-- <p>Aux Id: Seq 10966</p> --> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.12.1 --> + + <section> + <title>SNMP Development Toolkit 4.12</title> + <p>Version 4.12 supports code replacement in runtime from/to + version 4.11.2, 4.11.1 and 4.11.</p> + + <section> + <title>Improvements and new features</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[agent] A simple lookup cache has been added to improve + the mib server lookup performance. </p> + <p>This can be disabled with the mib_server + <seealso marker="snmp_app">cache</seealso> option. </p> + <p>Own Id: OTP-7346</p> + </item> + + <item> + <p>[agent] Improvement of the inform reporting. + It was previously not certain how many acks an + application received, 0, 1 or 2. This has now been + fixed, so that only 1 (one) ack is issued. </p> + <p>Per Hedeland</p> + <p>Own Id: OTP-7525</p> + </item> + + </list> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <p>-</p> + <!-- + <list type="bulleted"> + <item> + <p>[manager] Encryption error when attempting to send + version 3 inform-requests. </p> + <p>Own Id: OTP-7432</p> + <p>Aux Id: Seq 10966</p> + </item> + + </list> + --> + + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.12 --> + + <section> + <title>SNMP Development Toolkit 4.11.2</title> + <p>Version 4.11.2 supports code replacement in runtime from/to + version 4.11.1 and 4.11. </p> + + <section> + <title>Improvements and new features</title> + <p>-</p> + <!-- + <list type="bulleted"> + <item> + <p>Added utility functions for transforming DateAndTime + as [int()] to strings; + <seealso marker="snmp#dat2s">date_and_time_to_string/2</seealso> + and + <seealso marker="snmp#dat2s2">date_and_time_to_string2/1</seealso>. </p> + <p>Also added new validation function + <seealso marker="snmp#vdat">validate_date_and_time/2</seealso>. </p> + <p>Own Id: OTP-7412</p> + <p>Aux Id: Seq 10987</p> + </item> + </list> + --> + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[manager] Erroneous engine-id check when receiving version 3 + informs. </p> + <p>Own Id: OTP-7570</p> + <p>Aux Id: Seq 11060</p> + </item> + + <item> + <p>Receiving an snmp message with a very large version + number could cause the erlang node to run out of + memory and consequently crash. </p> + <p>The standard specifies the snmp version as an + (unlimited) INTEGER, but today only + 0 (version 1), 1 (version 2) and 3 (version 3) is + actually used. So, when decoding a message, a limit + has been put on the snmp version integer in order + to not allow this kind of a problem. </p> + <p>Own Id: OTP-7575</p> + <p>Aux Id: Seq 11064</p> + </item> + + </list> + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.11.2 --> + + + <section> + <title>SNMP Development Toolkit 4.11.1</title> + <p>Version 4.11.1 supports code replacement in runtime from/to + version 4.11.</p> + + <section> + <title>Improvements and new features</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[compiler] The MIB compiler did not retrieve the REFERENCE part + of a SNMP MIB definition. </p> + <p>This problem has been partly solved. For SNMP tables, + the assocList field of the tables mib-entry record now contains + this info (as <c>{reference, string()}</c>), <em>if</em> the + MIB was compiled with the compiler option <em>+reference</em>. </p> + <p>This solution is temporary, until such time as a permanent + solution (and probably not backward compatible) is devised, which + retrieves and stores all REFERENCE part(s) of a MIB. </p> + <p>See the + <seealso marker="snmpc#compiler_opts">compiler options</seealso> + for more info. </p> + + <p>Serge Aleynikov</p> + <p>Own Id: OTP-7426</p> + </item> + + <item> + <p>Added utility functions for transforming DateAndTime + as [int()] to strings; + <seealso marker="snmp#dat2s">date_and_time_to_string/2</seealso> + and + <seealso marker="snmp#dat2s2">date_and_time_to_string2/1</seealso>. </p> + <p>Also added new validation function + <seealso marker="snmp#vdat">validate_date_and_time/2</seealso>. </p> + <p>Own Id: OTP-7412</p> + <p>Aux Id: Seq 10987</p> + </item> + + </list> + + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[manager] Encryption error when attempting to send + version 3 inform-requests. </p> + <p>Own Id: OTP-7432</p> + <p>Aux Id: Seq 10966</p> + </item> + + </list> + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.11.1 --> + + <section> + <title>SNMP Development Toolkit 4.11</title> + <p>Version 4.11 supports code replacement in runtime from/to + version 4.10.3, 4.10.2, 4.10.1 and 4.10.</p> + + <section> + <title>Improvements and new features</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[agent] Performance improvements in the case when an SNMP + manager performs an snmpwalk. </p> + <p>Martin Björklund</p> + <p>Own Id: OTP-7201</p> + </item> + + <item> + <p>The API for sending inform(s) has been improved. Also + the documentation has been corrected and updated. See + <seealso marker="snmpa#send_notification">snmpa:send_notification</seealso> and + <seealso marker="snmpa_notification_delivery_info_receiver">snmpa_notification_delivery_info_receiver</seealso> + for more info.</p> + <p>Own Id: OTP-7287</p> + <p>Aux Id: Seq 10926</p> + </item> + + <item> + <p>[agent] Performance of the internal database (local-db) + has been improved.</p> + <p>Own Id: OTP-7319</p> + <p>Aux Id: Seq 10942</p> + </item> + + <item> + <p>[agent] Added utility functions, + <seealso marker="snmpa#restart_worker">snmpa:restart_worker/0,1</seealso> and + <seealso marker="snmpa#restart_set_worker">snmpa:restart_set_worker/0,1</seealso>, + for restarting the agent worker processes (in case the agent is + multi-threaded).</p> + <p>Own Id: OTP-7369</p> + </item> + + <item> + <p>Add utility function to + <seealso marker="snmp#read_mib">read</seealso> + a compiled mib. </p> + <p>Own Id: OTP-7371</p> + </item> + + </list> + </section> + + <section> + <title>Reported Fixed Bugs and Malfunctions</title> + <!-- + <p>-</p> + --> + <list type="bulleted"> + <item> + <p>[manager] Encryption error when attempting to send + version 3 inform-requests. </p> + <p>Own Id: OTP-7377</p> + <p>Aux Id: Seq 10966</p> + </item> + + </list> + </section> + + <section> + <title>Incompatibilities</title> + <p>-</p> + </section> + </section> <!-- 4.11 --> + + + <section> <title>SNMP Development Toolkit 4.10.3</title> <p>Version 4.10.3 supports code replacement in runtime from/to version 4.10.2, 4.10.1 and 4.10.</p> diff --git a/lib/snmp/src/app/snmp.appup.src b/lib/snmp/src/app/snmp.appup.src index 0b6ea93231..af988fda26 100644 --- a/lib/snmp/src/app/snmp.appup.src +++ b/lib/snmp/src/app/snmp.appup.src @@ -22,8 +22,18 @@ %% ----- U p g r a d e ------------------------------------------------------- [ + {"4.21.2", + [ + ] + }, + {"4.21.1", + [ + {update, snmp_note_store, soft, soft_purge, soft_purge, []} + ] + }, {"4.21", [ + {update, snmp_note_store, soft, soft_purge, soft_purge, []}, {load_module, snmp_target_mib, soft_purge, soft_purge, []} ] }, @@ -40,6 +50,7 @@ {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf, snmp_config]}, {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, + {update, snmp_note_store, soft, soft_purge, soft_purge, []}, {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, {update, snmpm_server, soft, soft_purge, soft_purge, @@ -47,76 +58,24 @@ {update, snmpm_net_if, soft, soft_purge, soft_purge, [snmp_conf, snmpm_mpd, snmpm_config]} ] - }, - {"4.20", - [ - {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, - {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]}, - {load_module, snmpm, soft_purge, soft_purge, - [snmpm_server, snmpm_config, snmp_config]}, - {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmp_config, soft_purge, soft_purge, []}, - {load_module, snmpm_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config, snmpm_config]}, - {load_module, snmpa_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config]}, - {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, - {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, - {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, - {update, snmpm_server, soft, soft_purge, soft_purge, - [snmpm_net_if, snmpm_mpd, snmpm_config]}, - {update, snmpm_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpm_mpd, snmpm_config]} - ] - }, - {"4.19", - [ - {load_module, snmpa, soft_purge, soft_purge, []}, - {load_module, snmpm, soft_purge, soft_purge, - [snmpm_server, snmpm_config, snmp_config]}, - {load_module, snmpa_usm, soft_purge, soft_purge, []}, - {load_module, snmpm_usm, soft_purge, soft_purge, []}, - {load_module, snmp_log, soft_purge, soft_purge, []}, - {load_module, snmp_pdus, soft_purge, soft_purge, []}, - {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmpa_conf, soft_purge, soft_purge, - [snmp_conf, snmp_config]}, - {load_module, snmp_misc, soft_purge, soft_purge, []}, - {load_module, snmp_config, soft_purge, soft_purge, []}, - {load_module, snmpa_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config]}, - {load_module, snmpm_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config, snmpm_config]}, - {load_module, snmpa_trap, soft_purge, soft_purge, - [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]}, - {load_module, snmpa_acm, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd, snmp_target_mib]}, - {load_module, snmpa_conf, soft_purge, soft_purge, - [snmp_config, snmp_notification_mib]}, - {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, - {load_module, snmp_notification_mib, soft_purge, soft_purge, - [snmp_conf, snmp_target_mib]}, - {load_module, snmp_community_mib, soft_purge, soft_purge, []}, - {load_module, snmp_target_mib, soft_purge, soft_purge, - [snmp_conf]}, - {update, snmpm_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpm_mpd, snmpm_config]}, - {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, - {update, snmpm_server, soft, soft_purge, soft_purge, - [snmpm_net_if, snmpm_mpd, snmpm_config]}, - {update, snmpa_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd]}, - {update, snmpa_agent, soft, soft_purge, soft_purge, - [snmpa_acm, snmpa_mpd, snmpa_trap]} - ] - } + } ], %% ------D o w n g r a d e --------------------------------------------------- [ + {"4.21.2", + [ + ] + }, + {"4.21.1", + [ + {update, snmp_note_store, soft, soft_purge, soft_purge, []} + ] + }, {"4.21", [ + {update, snmp_note_store, soft, soft_purge, soft_purge, []}, {load_module, snmp_target_mib, soft_purge, soft_purge, []} ] }, @@ -133,6 +92,7 @@ {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf, snmp_config]}, {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, + {update, snmp_note_store, soft, soft_purge, soft_purge, []}, {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, {update, snmpm_server, soft, soft_purge, soft_purge, @@ -140,68 +100,6 @@ {update, snmpm_net_if, soft, soft_purge, soft_purge, [snmp_conf, snmpm_mpd, snmpm_config]} ] - }, - {"4.20", - [ - {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, - {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]}, - {load_module, snmpm, soft_purge, soft_purge, - [snmpm_server, snmpm_config, snmp_config]}, - {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmp_config, soft_purge, soft_purge, []}, - {load_module, snmpm_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config, snmpm_config]}, - {load_module, snmpa_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config]}, - {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]}, - {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]}, - {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, - {update, snmpm_server, soft, soft_purge, soft_purge, - [snmpm_net_if, snmpm_mpd, snmpm_config]}, - {update, snmpm_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpm_mpd, snmpm_config]} - ] - }, - {"4.19", - [ - {load_module, snmpa, soft_purge, soft_purge, []}, - {load_module, snmpm, soft_purge, soft_purge, - [snmpm_server, snmpm_config, snmp_config]}, - {load_module, snmpa_usm, soft_purge, soft_purge, []}, - {load_module, snmpm_usm, soft_purge, soft_purge, []}, - {load_module, snmp_log, soft_purge, soft_purge, []}, - {load_module, snmp_pdus, soft_purge, soft_purge, []}, - {load_module, snmp_conf, soft_purge, soft_purge, []}, - {load_module, snmpa_conf, soft_purge, soft_purge, - [snmp_conf, snmp_config]}, - {load_module, snmp_misc, soft_purge, soft_purge, []}, - {load_module, snmp_config, soft_purge, soft_purge, []}, - {load_module, snmpa_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config]}, - {load_module, snmpm_mpd, soft_purge, soft_purge, - [snmp_conf, snmp_config, snmpm_config]}, - {load_module, snmpa_trap, soft_purge, soft_purge, - [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]}, - {load_module, snmpa_acm, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd, snmp_target_mib]}, - {load_module, snmpa_conf, soft_purge, soft_purge, - [snmp_config, snmp_notification_mib]}, - {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []}, - {load_module, snmp_notification_mib, soft_purge, soft_purge, - [snmp_conf, snmp_target_mib]}, - {load_module, snmp_community_mib, soft_purge, soft_purge, []}, - {load_module, snmp_target_mib, soft_purge, soft_purge, - [snmp_conf]}, - {update, snmpm_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpm_mpd, snmpm_config]}, - {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]}, - {update, snmpm_server, soft, soft_purge, soft_purge, - [snmpm_net_if, snmpm_mpd, snmpm_config]}, - {update, snmpa_net_if, soft, soft_purge, soft_purge, - [snmp_conf, snmpa_mpd]}, - {update, snmpa_agent, soft, soft_purge, soft_purge, - [snmpa_acm, snmpa_mpd, snmpa_trap]} - ] } ] }. diff --git a/lib/snmp/src/compile/depend.mk b/lib/snmp/src/compile/depend.mk index f7084f8bcd..3ee8dc4bec 100644 --- a/lib/snmp/src/compile/depend.mk +++ b/lib/snmp/src/compile/depend.mk @@ -44,6 +44,6 @@ $(EBIN)/snmpc_mib_gram.$(EMULATOR): \ ../../include/snmp_types.hrl \ snmpc_mib_gram.erl -$(BIN)/snmpc: snmpc.src +$(BIN)/snmpc: snmpc.src ../../vsn.mk $(PERL) -p -e 's?%VSN%?$(VSN)? ' < $< > $@ chmod 755 $@ diff --git a/lib/snmp/src/compile/snmpc.src b/lib/snmp/src/compile/snmpc.src index f993335b89..868e0929b4 100644 --- a/lib/snmp/src/compile/snmpc.src +++ b/lib/snmp/src/compile/snmpc.src @@ -74,7 +74,7 @@ %% --rrnac %% --version %% --verbosity V -%% --warnings +%% --warnings | --W %% --Werror | --wae | --warnings_as_errors main(Args) when is_list(Args) -> case (catch process_args(Args)) of @@ -221,7 +221,10 @@ process_args([], #state{verbosity = Verbosity0, file = MIB} = State) -> process_args(["--help"|_Args], _State) -> ok; process_args(["--version"|_Args], #state{version = Version, mfv = MFV} = _State) -> - {ok, lists:flatten(io_lib:format("snmpc ~s (~s)", [Version, MFV]))}; + OtpVersion = otp_release(), + {ok, lists:flatten( + io_lib:format("snmpc ~s [Mib format version ~s] (OTP ~s)", + [Version, MFV, OtpVersion]))}; process_args(["--verbosity", Verbosity0|Args], #state{verbosity = V} = State) when (V =:= undefined) -> Verbosity = list_to_atom(Verbosity0), @@ -234,7 +237,7 @@ process_args(["--verbosity", Verbosity0|Args], #state{verbosity = V} = State) process_args(["--verbosity"|_Args], #state{verbosity = V}) when (V =/= undefined) -> e(lists:flatten(io_lib:format("Verbosity already set to ~w", [V]))); -process_args(["--w"|Args], State) -> +process_args(["--W"|Args], State) -> process_args(Args, State#state{warnings = true}); process_args(["--warnings"|Args], State) -> process_args(Args, State#state{warnings = true}); @@ -398,3 +401,17 @@ usage() -> e(Reason) -> throw({error, Reason}). + +otp_release() -> + system_info(otp_release, string). + +system_info(Tag, Type) -> + case (catch erlang:system_info(Tag)) of + {'EXIT', _} -> + "-"; + Info when is_list(Info) andalso (Type =:= string) -> + Info; + Info -> + lists:flatten(io_lib:format("~w", [Info])) + end. + diff --git a/lib/snmp/src/misc/snmp_note_store.erl b/lib/snmp/src/misc/snmp_note_store.erl index a21a6209f1..23fccf8a5f 100644 --- a/lib/snmp/src/misc/snmp_note_store.erl +++ b/lib/snmp/src/misc/snmp_note_store.erl @@ -258,10 +258,17 @@ code_change({down, _Vsn}, State, _Extra) -> {ok, NState}; % upgrade -code_change(_Vsn, State, _Extra) -> +code_change(_Vsn, State0, _Extra) -> process_flag(trap_exit, true), - NState = restart_timer(State), - {ok, NState}. + State1 = + case State0 of + #state{timeout = false} -> + State0#state{timeout = ?timeout}; + _ -> + State0 + end, + State2 = restart_timer(State1), + {ok, State2}. %%---------------------------------------------------------- @@ -282,7 +289,7 @@ deactivate_timer(#state{timer = Pid, active = true} = State) -> receive deactivated -> ok end, - State#state{timeout = false}; + State#state{active = false}; deactivate_timer(State) -> State. diff --git a/lib/snmp/test/snmp_manager_user_old.erl b/lib/snmp/test/snmp_manager_user_old.erl index edffc80dd4..edffc80dd4 100755..100644 --- a/lib/snmp/test/snmp_manager_user_old.erl +++ b/lib/snmp/test/snmp_manager_user_old.erl diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index c95e0a22d1..25e3a9470b 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -17,6 +17,6 @@ # # %CopyrightEnd% -SNMP_VSN = 4.21.1 +SNMP_VSN = 4.21.3 PRE_VSN = APP_VSN = "snmp-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/src/DSS.asn1 b/lib/ssh/src/DSS.asn1 index 77aca3808b..77aca3808b 100755..100644 --- a/lib/ssh/src/DSS.asn1 +++ b/lib/ssh/src/DSS.asn1 diff --git a/lib/ssh/src/PKCS-1.asn1 b/lib/ssh/src/PKCS-1.asn1 index e7d6b18c63..e7d6b18c63 100755..100644 --- a/lib/ssh/src/PKCS-1.asn1 +++ b/lib/ssh/src/PKCS-1.asn1 diff --git a/lib/ssh/src/ssh_bits.erl b/lib/ssh/src/ssh_bits.erl index 3f0a06575c..3f0a06575c 100755..100644 --- a/lib/ssh/src/ssh_bits.erl +++ b/lib/ssh/src/ssh_bits.erl diff --git a/lib/ssh/src/ssh_connect.hrl b/lib/ssh/src/ssh_connect.hrl index 34d4ff8fc1..34d4ff8fc1 100755..100644 --- a/lib/ssh/src/ssh_connect.hrl +++ b/lib/ssh/src/ssh_connect.hrl diff --git a/lib/ssh/src/ssh_dsa.erl b/lib/ssh/src/ssh_dsa.erl index 1b9a396f0c..1b9a396f0c 100755..100644 --- a/lib/ssh/src/ssh_dsa.erl +++ b/lib/ssh/src/ssh_dsa.erl diff --git a/lib/ssh/src/ssh_file.erl b/lib/ssh/src/ssh_file.erl index 12180f56bb..12180f56bb 100755..100644 --- a/lib/ssh/src/ssh_file.erl +++ b/lib/ssh/src/ssh_file.erl diff --git a/lib/ssh/src/ssh_io.erl b/lib/ssh/src/ssh_io.erl index 915fd63e4f..915fd63e4f 100755..100644 --- a/lib/ssh/src/ssh_io.erl +++ b/lib/ssh/src/ssh_io.erl diff --git a/lib/ssh/src/ssh_math.erl b/lib/ssh/src/ssh_math.erl index 510eb16aa6..510eb16aa6 100755..100644 --- a/lib/ssh/src/ssh_math.erl +++ b/lib/ssh/src/ssh_math.erl diff --git a/lib/ssh/src/ssh_rsa.erl b/lib/ssh/src/ssh_rsa.erl index 91b8285b2e..91b8285b2e 100755..100644 --- a/lib/ssh/src/ssh_rsa.erl +++ b/lib/ssh/src/ssh_rsa.erl diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index f000558100..f000558100 100755..100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl diff --git a/lib/ssh/src/ssh_userauth.hrl b/lib/ssh/src/ssh_userauth.hrl index 8eb2d46ed1..8eb2d46ed1 100755..100644 --- a/lib/ssh/src/ssh_userauth.hrl +++ b/lib/ssh/src/ssh_userauth.hrl diff --git a/lib/ssh/src/ssh_xfer.hrl b/lib/ssh/src/ssh_xfer.hrl index 4a4f1a4291..4a4f1a4291 100755..100644 --- a/lib/ssh/src/ssh_xfer.hrl +++ b/lib/ssh/src/ssh_xfer.hrl diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 70122e4393..50268ae206 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -71,7 +71,8 @@ {fail_if_no_peer_cert, boolean()} {depth, integer()} | {cert, der_encoded()}| {certfile, path()} | - {key, der_encoded()} | {keyfile, path()} | {password, string()} | + {key, {'RSAPrivateKey'| 'DSAPrivateKey' | 'PrivateKeyInfo', der_encoded()}} | + {keyfile, path()} | {password, string()} | {cacerts, [der_encoded()]} | {cacertfile, path()} | |{dh, der_encoded()} | {dhfile, path()} | {ciphers, ciphers()} | {ssl_imp, ssl_imp()} | {reuse_sessions, boolean()} | {reuse_session, fun()} @@ -139,7 +140,7 @@ <tag>{certfile, path()}</tag> <item>Path to a file containing the user's certificate.</item> - <tag>{key, der_encoded()}</tag> + <tag>{key, {'RSAPrivateKey'| 'DSAPrivateKey' | 'PrivateKeyInfo', der_encoded()}}</tag> <item> The DER encoded users private key. If this option is supplied it will override the keyfile option.</item> diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index 29674f30da..1b07e76d6a 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,6 +1,7 @@ %% -*- erlang -*- {"%VSN%", [ + {"4.1.6", [{restart_application, ssl}]}, {"4.1.5", [{restart_application, ssl}]}, {"4.1.4", [{restart_application, ssl}]}, {"4.1.3", [{restart_application, ssl}]}, @@ -10,6 +11,7 @@ {"4.0.1", [{restart_application, ssl}]} ], [ + {"4.1.6", [{restart_application, ssl}]}, {"4.1.5", [{restart_application, ssl}]}, {"4.1.4", [{restart_application, ssl}]}, {"4.1.3", [{restart_application, ssl}]}, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 35f9410562..d0693445e0 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -608,8 +608,11 @@ validate_option(certfile, Value) when Value == undefined; is_list(Value) -> validate_option(key, undefined) -> undefined; validate_option(key, {KeyType, Value}) when is_binary(Value), - KeyType == rsa; - KeyType == dsa -> + KeyType == rsa; %% Backwards compatibility + KeyType == dsa; %% Backwards compatibility + KeyType == 'RSAPrivateKey'; + KeyType == 'DSAPrivateKey'; + KeyType == 'PrivateKeyInfo' -> {KeyType, Value}; validate_option(keyfile, Value) when is_list(Value) -> Value; diff --git a/lib/ssl/src/ssl_certificate.erl b/lib/ssl/src/ssl_certificate.erl index 422ea6404b..61876e1158 100644 --- a/lib/ssl/src/ssl_certificate.erl +++ b/lib/ssl/src/ssl_certificate.erl @@ -66,7 +66,7 @@ trusted_cert_and_path(CertChain, CertDbHandle, CertDbRef) -> {ok, IssuerId} -> {other, IssuerId}; {error, issuer_not_found} -> - case find_issuer(OtpCert, no_candidate, CertDbHandle) of + case find_issuer(OtpCert, CertDbHandle) of {ok, IssuerId} -> {other, IssuerId}; Other -> @@ -193,7 +193,7 @@ certificate_chain(OtpCert, _Cert, CertDbHandle, CertsDbRef, Chain) -> {_, true = SelfSigned} -> certificate_chain(CertDbHandle, CertsDbRef, Chain, ignore, ignore, SelfSigned); {{error, issuer_not_found}, SelfSigned} -> - case find_issuer(OtpCert, no_candidate, CertDbHandle) of + case find_issuer(OtpCert, CertDbHandle) of {ok, {SerialNr, Issuer}} -> certificate_chain(CertDbHandle, CertsDbRef, Chain, SerialNr, Issuer, SelfSigned); @@ -227,17 +227,24 @@ certificate_chain(CertDbHandle, CertsDbRef, Chain, SerialNr, Issuer, _SelfSigned {ok, lists:reverse(Chain)} end. -find_issuer(OtpCert, PrevCandidateKey, CertDbHandle) -> - case ssl_manager:issuer_candidate(PrevCandidateKey, CertDbHandle) of - no_more_candidates -> - {error, issuer_not_found}; - {Key, {_Cert, ErlCertCandidate}} -> - case public_key:pkix_is_issuer(OtpCert, ErlCertCandidate) of - true -> - public_key:pkix_issuer_id(ErlCertCandidate, self); - false -> - find_issuer(OtpCert, Key, CertDbHandle) - end +find_issuer(OtpCert, CertDbHandle) -> + IsIssuerFun = fun({_Key, {_Der, #'OTPCertificate'{} = ErlCertCandidate}}, Acc) -> + case public_key:pkix_is_issuer(OtpCert, ErlCertCandidate) of + true -> + throw(public_key:pkix_issuer_id(ErlCertCandidate, self)); + false -> + Acc + end; + (_, Acc) -> + Acc + end, + + try ssl_certificate_db:foldl(IsIssuerFun, issuer_not_found, CertDbHandle) of + issuer_not_found -> + {error, issuer_not_found} + catch + {ok, _IssuerId} = Return -> + Return end. is_valid_extkey_usage(KeyUse, client) -> diff --git a/lib/ssl/src/ssl_certificate_db.erl b/lib/ssl/src/ssl_certificate_db.erl index 0560a02110..cb2473576a 100644 --- a/lib/ssl/src/ssl_certificate_db.erl +++ b/lib/ssl/src/ssl_certificate_db.erl @@ -26,7 +26,7 @@ -include_lib("public_key/include/public_key.hrl"). -export([create/0, remove/1, add_trusted_certs/3, - remove_trusted_certs/2, lookup_trusted_cert/4, issuer_candidate/2, + remove_trusted_certs/2, lookup_trusted_cert/4, foldl/3, lookup_cached_certs/2, cache_pem_file/4, uncache_pem_file/2, lookup/2]). -type time() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}. @@ -127,8 +127,6 @@ uncache_pem_file(File, [_CertsDb, _FileToRefDb, PidToFileDb]) -> exit(Pid, shutdown) end, Pids). - - %%-------------------------------------------------------------------- -spec remove_trusted_certs(pid(), [db_handle()]) -> term(). @@ -161,37 +159,6 @@ remove_trusted_certs(Pid, [CertsDb, FileToRefDb, PidToFileDb]) -> end. %%-------------------------------------------------------------------- --spec issuer_candidate(no_candidate | cert_key() | {file, term()}, term()) -> - {cert_key(),{der_cert(), #'OTPCertificate'{}}} | no_more_candidates. -%% -%% Description: If a certificat does not define its issuer through -%% the extension 'ce-authorityKeyIdentifier' we can -%% try to find the issuer in the database over known -%% certificates. -%%-------------------------------------------------------------------- -issuer_candidate(no_candidate, Db) -> - case ets:first(Db) of - '$end_of_table' -> - no_more_candidates; - {file, _} = Key -> - issuer_candidate(Key, Db); - Key -> - [Cert] = lookup(Key, Db), - {Key, Cert} - end; - -issuer_candidate(PrevCandidateKey, Db) -> - case ets:next(Db, PrevCandidateKey) of - '$end_of_table' -> - no_more_candidates; - {file, _} = Key -> - issuer_candidate(Key, Db); - Key -> - [Cert] = lookup(Key, Db), - {Key, Cert} - end. - -%%-------------------------------------------------------------------- -spec lookup(term(), db_handle()) -> term() | undefined. %% %% Description: Looks up an element in a certificat <Db>. @@ -206,7 +173,18 @@ lookup(Key, Db) -> end, [Pick(Data) || Data <- Contents] end. - +%%-------------------------------------------------------------------- +-spec foldl(fun(), term(), db_handle()) -> term(). +%% +%% Description: Calls Fun(Elem, AccIn) on successive elements of the +%% cache, starting with AccIn == Acc0. Fun/2 must return a new +%% accumulator which is passed to the next call. The function returns +%% the final value of the accumulator. Acc0 is returned if the certifate +%% db is empty. +%%-------------------------------------------------------------------- +foldl(Fun, Acc0, Cache) -> + ets:foldl(Fun, Acc0, Cache). + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index c772697f1d..0c44d3ae90 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -303,12 +303,13 @@ init([Role, Host, Port, Socket, {SSLOpts0, _} = Options, User, CbInfo]) -> State0 = initial_state(Role, Host, Port, Socket, Options, User, CbInfo), Hashes0 = ssl_handshake:init_hashes(), - + TimeStamp = calendar:datetime_to_gregorian_seconds({date(), time()}), try ssl_init(SSLOpts0, Role) of {ok, Ref, CertDbHandle, CacheHandle, OwnCert, Key, DHParams} -> Session = State0#state.session, State = State0#state{tls_handshake_hashes = Hashes0, - session = Session#session{own_certificate = OwnCert}, + session = Session#session{own_certificate = OwnCert, + time_stamp = TimeStamp}, cert_db_ref = Ref, cert_db = CertDbHandle, session_cache = CacheHandle, @@ -351,8 +352,7 @@ hello(start, #state{host = Host, port = Port, role = client, State1 = State0#state{connection_states = CS2, negotiated_version = Version, %% Requested version session = - Session0#session{session_id = Hello#client_hello.session_id, - is_resumable = false}, + Session0#session{session_id = Hello#client_hello.session_id}, tls_handshake_hashes = Hashes1}, {Record, State} = next_record(State1), next_state(hello, Record, State); @@ -1126,18 +1126,38 @@ init_private_key(DbHandle, undefined, KeyFile, Password, _) -> {ok, List} = ssl_manager:cache_pem_file(KeyFile, DbHandle), [PemEntry] = [PemEntry || PemEntry = {PKey, _ , _} <- List, PKey =:= 'RSAPrivateKey' orelse - PKey =:= 'DSAPrivateKey'], - public_key:pem_entry_decode(PemEntry, Password) + PKey =:= 'DSAPrivateKey' orelse + PKey =:= 'PrivateKeyInfo' + ], + private_key(public_key:pem_entry_decode(PemEntry, Password)) catch Error:Reason -> handle_file_error(?LINE, Error, Reason, KeyFile, ekeyfile, erlang:get_stacktrace()) end; +%% First two clauses are for backwards compatibility init_private_key(_,{rsa, PrivateKey}, _, _,_) -> - public_key:der_decode('RSAPrivateKey', PrivateKey); + init_private_key('RSAPrivateKey', PrivateKey); init_private_key(_,{dsa, PrivateKey},_,_,_) -> - public_key:der_decode('DSAPrivateKey', PrivateKey). + init_private_key('DSAPrivateKey', PrivateKey); +init_private_key(_,{Asn1Type, PrivateKey},_,_,_) -> + private_key(init_private_key(Asn1Type, PrivateKey)). + +init_private_key(Asn1Type, PrivateKey) -> + public_key:der_decode(Asn1Type, PrivateKey). + +private_key(#'PrivateKeyInfo'{privateKeyAlgorithm = + #'PrivateKeyInfo_privateKeyAlgorithm'{algorithm = ?'rsaEncryption'}, + privateKey = Key}) -> + public_key:der_decode('RSAPrivateKey', iolist_to_binary(Key)); + +private_key(#'PrivateKeyInfo'{privateKeyAlgorithm = + #'PrivateKeyInfo_privateKeyAlgorithm'{algorithm = ?'id-dsa'}, + privateKey = Key}) -> + public_key:der_decode('DSAPrivateKey', iolist_to_binary(Key)); +private_key(Key) -> + Key. -spec(handle_file_error(_,_,_,_,_,_) -> no_return()). handle_file_error(Line, Error, {badmatch, Reason}, File, Throw, Stack) -> @@ -1987,16 +2007,16 @@ next_state_is_connection(State0) -> public_key_info = undefined, tls_handshake_hashes = {<<>>, <<>>}}). -register_session(_, _, _, #session{is_resumable = true} = Session) -> - Session; %% Already registered -register_session(client, Host, Port, Session0) -> +register_session(client, Host, Port, #session{is_resumable = new} = Session0) -> Session = Session0#session{is_resumable = true}, ssl_manager:register_session(Host, Port, Session), Session; -register_session(server, _, Port, Session0) -> +register_session(server, _, Port, #session{is_resumable = new} = Session0) -> Session = Session0#session{is_resumable = true}, ssl_manager:register_session(Port, Session), - Session. + Session; +register_session(_, _, _, Session) -> + Session. %% Already registered invalidate_session(client, Host, Port, Session) -> ssl_manager:invalidate_session(Host, Port, Session); @@ -2020,7 +2040,7 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions}, User, %% We do not want to save the password in the state so that %% could be written in the clear into error logs. ssl_options = SSLOptions#ssl_options{password = undefined}, - session = #session{is_resumable = false}, + session = #session{is_resumable = new}, transport_cb = CbModule, data_tag = DataTag, close_tag = CloseTag, diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index f873a6a913..7eb7f44df6 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -1092,18 +1092,12 @@ certificate_authorities(CertDbHandle, CertDbRef) -> list_to_binary([Enc(Cert) || {_, Cert} <- Authorities]). certificate_authorities_from_db(CertDbHandle, CertDbRef) -> - certificate_authorities_from_db(CertDbHandle, CertDbRef, no_candidate, []). - -certificate_authorities_from_db(CertDbHandle,CertDbRef, PrevKey, Acc) -> - case ssl_manager:issuer_candidate(PrevKey, CertDbHandle) of - no_more_candidates -> - lists:reverse(Acc); - {{CertDbRef, _, _} = Key, Cert} -> - certificate_authorities_from_db(CertDbHandle, CertDbRef, Key, [Cert|Acc]); - {Key, _Cert} -> - %% skip certs not from this ssl connection - certificate_authorities_from_db(CertDbHandle, CertDbRef, Key, Acc) - end. + ConnectionCerts = fun({{Ref, _, _}, Cert}, Acc) when Ref == CertDbRef -> + [Cert | Acc]; + (_, Acc) -> + Acc + end, + ssl_certificate_db:foldl(ConnectionCerts, [], CertDbHandle). digitally_signed(Hash, #'RSAPrivateKey'{} = Key) -> public_key:encrypt_private(Hash, Key, diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl index dcf310c535..6a44ef8c3e 100644 --- a/lib/ssl/src/ssl_manager.erl +++ b/lib/ssl/src/ssl_manager.erl @@ -29,8 +29,8 @@ %% Internal application API -export([start_link/1, start_link_dist/1, connection_init/2, cache_pem_file/2, - lookup_trusted_cert/4, issuer_candidate/2, client_session_id/4, - server_session_id/4, + lookup_trusted_cert/4, + client_session_id/4, server_session_id/4, register_session/2, register_session/3, invalidate_session/2, invalidate_session/3]). @@ -112,16 +112,7 @@ cache_pem_file(File, DbHandle) -> %% -------------------------------------------------------------------- lookup_trusted_cert(DbHandle, Ref, SerialNumber, Issuer) -> ssl_certificate_db:lookup_trusted_cert(DbHandle, Ref, SerialNumber, Issuer). -%%-------------------------------------------------------------------- --spec issuer_candidate(cert_key() | no_candidate, term()) -> - {cert_key(), - {der_cert(), - #'OTPCertificate'{}}} | no_more_candidates. -%% -%% Description: Return next issuer candidate. -%%-------------------------------------------------------------------- -issuer_candidate(PrevCandidateKey, DbHandle) -> - ssl_certificate_db:issuer_candidate(PrevCandidateKey, DbHandle). + %%-------------------------------------------------------------------- -spec client_session_id(host(), inet:port_number(), #ssl_options{}, der_cert() | undefined) -> session_id(). @@ -278,25 +269,16 @@ handle_cast({register_session, Port, Session}, CacheCb:update(Cache, {Port, NewSession#session.session_id}, NewSession), {noreply, State}; -%%% When a session is invalidated we need to wait a while before deleting -%%% it as there might be pending connections that rightfully needs to look -%%% up the session data but new connections should not get to use this session. handle_cast({invalidate_session, Host, Port, #session{session_id = ID} = Session}, #state{session_cache = Cache, session_cache_cb = CacheCb} = State) -> - CacheCb:update(Cache, {{Host, Port}, ID}, Session#session{is_resumable = false}), - TRef = - erlang:send_after(delay_time(), self(), {delayed_clean_session, {{Host, Port}, ID}}), - {noreply, State#state{last_delay_timer = TRef}}; + invalidate_session(Cache, CacheCb, {{Host, Port}, ID}, Session, State); handle_cast({invalidate_session, Port, #session{session_id = ID} = Session}, #state{session_cache = Cache, session_cache_cb = CacheCb} = State) -> - CacheCb:update(Cache, {Port, ID}, Session#session{is_resumable = false}), - TRef = - erlang:send_after(delay_time(), self(), {delayed_clean_session, {Port, ID}}), - {noreply, State#state{last_delay_timer = TRef}}; + invalidate_session(Cache, CacheCb, {Port, ID}, Session, State); handle_cast({recache_pem, File, LastWrite, Pid, From}, #state{certificate_db = [_, FileToRefDb, _]} = State0) -> @@ -320,7 +302,7 @@ handle_cast({recache_pem, File, LastWrite, Pid, From}, %% {stop, reason(), #state{}}. %% %% Description: Handling all non call/cast messages -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------- handle_info(validate_sessions, #state{session_cache_cb = CacheCb, session_cache = Cache, session_lifetime = LifeTime @@ -444,3 +426,20 @@ delay_time() -> _ -> ?CLEAN_SESSION_DB end. + +invalidate_session(Cache, CacheCb, Key, Session, State) -> + case CacheCb:lookup(Cache, Key) of + undefined -> %% Session is already invalidated + {noreply, State}; + #session{is_resumable = new} -> + CacheCb:delete(Cache, Key), + {noreply, State}; + _ -> + %% When a registered session is invalidated we need to wait a while before deleting + %% it as there might be pending connections that rightfully needs to look + %% up the session data but new connections should not get to use this session. + CacheCb:update(Cache, Key, Session#session{is_resumable = false}), + TRef = + erlang:send_after(delay_time(), self(), {delayed_clean_session, Key}), + {noreply, State#state{last_delay_timer = TRef}} + end. diff --git a/lib/ssl/src/ssl_session.erl b/lib/ssl/src/ssl_session.erl index bf738649f6..df5d7e0146 100644 --- a/lib/ssl/src/ssl_session.erl +++ b/lib/ssl/src/ssl_session.erl @@ -103,9 +103,9 @@ select_session([], _, _) -> select_session(Sessions, #ssl_options{ciphers = Ciphers, reuse_sessions = ReuseSession}, OwnCert) -> - IsResumable = - fun(Session) -> - ReuseSession andalso (Session#session.is_resumable) andalso + IsResumable = + fun(Session) -> + ReuseSession andalso resumable(Session#session.is_resumable) andalso lists:member(Session#session.cipher_suite, Ciphers) andalso (OwnCert == Session#session.own_certificate) end, @@ -147,10 +147,10 @@ is_resumable(SuggestedSessionId, Port, ReuseEnabled, ReuseFun, Cache, #session{cipher_suite = CipherSuite, own_certificate = SessionOwnCert, compression_method = Compression, - is_resumable = Is_resumable, + is_resumable = IsResumable, peer_certificate = PeerCert} = Session -> ReuseEnabled - andalso Is_resumable + andalso resumable(IsResumable) andalso (OwnCert == SessionOwnCert) andalso valid_session(Session, SecondLifeTime) andalso ReuseFun(SuggestedSessionId, PeerCert, @@ -158,3 +158,8 @@ is_resumable(SuggestedSessionId, Port, ReuseEnabled, ReuseFun, Cache, undefined -> false end. + +resumable(new) -> + false; +resumable(IsResumable) -> + IsResumable. diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index a9109c5a6e..d9cb8002ed 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -2592,7 +2592,7 @@ client_renegotiate(Config) when is_list(Config) -> {options, ServerOpts}]), Port = ssl_test_lib:inet_port(Server), - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {?MODULE, @@ -2784,7 +2784,7 @@ extended_key_usage_verify_peer(Config) when is_list(Config) -> KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = public_key:pem_entry_decode(KeyEntry), + Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ServerCertFile = proplists:get_value(certfile, ServerOpts), NewServerCertFile = filename:join(PrivDir, "server/new_cert.pem"), @@ -2846,7 +2846,7 @@ extended_key_usage_verify_none(Config) when is_list(Config) -> KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = public_key:pem_entry_decode(KeyEntry), + Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ServerCertFile = proplists:get_value(certfile, ServerOpts), NewServerCertFile = filename:join(PrivDir, "server/new_cert.pem"), @@ -2908,7 +2908,7 @@ no_authority_key_identifier(Config) when is_list(Config) -> KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = public_key:pem_entry_decode(KeyEntry), + Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), CertFile = proplists:get_value(certfile, ServerOpts), NewCertFile = filename:join(PrivDir, "server/new_cert.pem"), @@ -2966,7 +2966,7 @@ invalid_signature_server(Config) when is_list(Config) -> KeyFile = filename:join(PrivDir, "server/key.pem"), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = public_key:pem_entry_decode(KeyEntry), + Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ServerCertFile = proplists:get_value(certfile, ServerOpts), NewServerCertFile = filename:join(PrivDir, "server/invalid_cert.pem"), @@ -3006,7 +3006,7 @@ invalid_signature_client(Config) when is_list(Config) -> KeyFile = filename:join(PrivDir, "client/key.pem"), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = public_key:pem_entry_decode(KeyEntry), + Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ClientCertFile = proplists:get_value(certfile, ClientOpts), NewClientCertFile = filename:join(PrivDir, "client/invalid_cert.pem"), @@ -3038,7 +3038,8 @@ tcp_delivery_workaround(Server, ServerMsg, Client, ClientMsg) -> {Client, ClientMsg} -> ok; {Client, {error,closed}} -> - test_server:format("client got close"); + test_server:format("client got close"), + ok; Unexpected -> test_server:fail(Unexpected) end; @@ -3083,7 +3084,7 @@ cert_expired(Config) when is_list(Config) -> KeyFile = filename:join(PrivDir, "otpCA/private/key.pem"), [KeyEntry] = ssl_test_lib:pem_to_der(KeyFile), - Key = public_key:pem_entry_decode(KeyEntry), + Key = ssl_test_lib:public_key(public_key:pem_entry_decode(KeyEntry)), ServerCertFile = proplists:get_value(certfile, ServerOpts), NewServerCertFile = filename:join(PrivDir, "server/expired_cert.pem"), @@ -3358,14 +3359,14 @@ der_input_opts(Opts) -> Keyfile = proplists:get_value(keyfile, Opts), Dhfile = proplists:get_value(dhfile, Opts), [{_, Cert, _}] = ssl_test_lib:pem_to_der(Certfile), - [{_, Key, _}] = ssl_test_lib:pem_to_der(Keyfile), + [{Asn1Type, Key, _}] = ssl_test_lib:pem_to_der(Keyfile), [{_, DHParams, _}] = ssl_test_lib:pem_to_der(Dhfile), CaCerts = lists:map(fun(Entry) -> {_, CaCert, _} = Entry, CaCert end, ssl_test_lib:pem_to_der(CaCertsfile)), - {Cert, {rsa, Key}, CaCerts, DHParams}. + {Cert, {Asn1Type, Key}, CaCerts, DHParams}. %%-------------------------------------------------------------------- %% different_ca_peer_sign(doc) -> @@ -3588,14 +3589,13 @@ hibernate(Config) -> {from, self()}, {mfa, {?MODULE, send_recv_result_active, []}}, {options, [{hibernate_after, 1000}|ClientOpts]}]), - - { current_function, { _M, _F, _A } } = + {current_function, _} = process_info(Pid, current_function), timer:sleep(1100), - { current_function, { erlang, hibernate, 3} } = - process_info(Pid, current_function), + {current_function, {erlang, hibernate, 3}} = + process_info(Pid, current_function), ssl_test_lib:close(Server), ssl_test_lib:close(Client). diff --git a/lib/ssl/test/ssl_cipher_SUITE.erl b/lib/ssl/test/ssl_cipher_SUITE.erl index 87478e13bc..99bc21e820 100644 --- a/lib/ssl/test/ssl_cipher_SUITE.erl +++ b/lib/ssl/test/ssl_cipher_SUITE.erl @@ -134,7 +134,7 @@ aes_decipher_good(Config) when is_list(Config) -> Version = {3,3}, Content = <<183,139,16,132,10,209,67,86,168,100,61,217,145,57,36,56,72,69,76,76,79,10>>, Mac = <<71,136,212,107,223,200,70,232,127,116,148,205,232,35,158,113,237,174,15,217,192,168,35,8,6,107,107,233,25,174,90,111>>, - {Content, Mac, CipherState1} = ssl_cipher:decipher(?AES, HashSz, CipherState, Fragment, Version), + {Content, Mac, _} = ssl_cipher:decipher(?AES, HashSz, CipherState, Fragment, Version), ok. %%-------------------------------------------------------------------- @@ -155,7 +155,7 @@ aes_decipher_fail(Config) when is_list(Config) -> 198,181,81,19,98,162,213,228,74,224,253,168,156,59,195,122, 108,101,107,242,20,15,169,150,163,107,101,94,93,104,241,165>>, Version = {3,3}, - {Content, Mac, CipherState1} = ssl_cipher:decipher(?AES, HashSz, CipherState, Fragment, Version), + {Content, Mac, _} = ssl_cipher:decipher(?AES, HashSz, CipherState, Fragment, Version), 32 = byte_size(Content), 32 = byte_size(Mac), ok. diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 23e9268f9b..8fe55ee7a4 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -54,9 +54,14 @@ end_per_group(_GroupName, Config) -> init_per_suite(Config0) -> try crypto:start() of ok -> - Config = add_ssl_opts_config(Config0), - setup_certs(Config), - Config + case test_server:is_cover() of + false -> + Config = add_ssl_opts_config(Config0), + setup_certs(Config), + Config; + true -> + {skip, "Can not be covered"} + end catch _:_ -> {skip, "Crypto did not start"} end. @@ -65,11 +70,31 @@ end_per_suite(Config) -> application:stop(crypto), Config. +init_per_testcase(plain_verify_options = Case, Config) when is_list(Config) -> + SslFlags = setup_dist_opts([{many_verify_opts, true} | Config]), + Flags = case os:getenv("ERL_FLAGS") of + false -> + os:putenv("ERL_FLAGS", SslFlags), + ""; + OldFlags -> + os:putenv("ERL_FLAGS", OldFlags ++ "" ++ SslFlags), + OldFlags + end, + common_init(Case, [{old_flags, Flags} | Config]); + init_per_testcase(Case, Config) when is_list(Config) -> + common_init(Case, Config). + +common_init(Case, Config) -> Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMETRAP_SECS)), [{watchdog, Dog},{testcase, Case}|Config]. -end_per_testcase(_Case, Config) when is_list(Config) -> +end_per_testcase(Case, Config) when is_list(Config) -> + Flags = proplists:get_value(old_flags, Config), + os:putenv("ERL_FLAGS", Flags), + common_end(Case, Config). + +common_end(_, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. @@ -205,9 +230,9 @@ plain_verify_options(Config) when is_list(Config) -> "server_reuse_sessions true client_reuse_sessions true " "server_hibernate_after 500 client_hibernate_after 500", - NH1 = start_ssl_node([{additional_dist_opts, DistOpts}, {many_verify_opts, true} | Config]), + NH1 = start_ssl_node([{additional_dist_opts, DistOpts} | Config]), Node1 = NH1#node_handle.nodename, - NH2 = start_ssl_node([{additional_dist_opts, DistOpts}, {many_verify_opts, true} | Config]), + NH2 = start_ssl_node([{additional_dist_opts, DistOpts} | Config]), Node2 = NH2#node_handle.nodename, pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end), @@ -467,8 +492,10 @@ cnct2tstsrvr([Host, Port]) when is_list(Host), is_list(Port) -> ets:insert(test_server_info, {test_server_handler, self()}), ssl_node_con_loop(Socket); - _Error -> - halt("Failed to connect to test server") + Error -> + halt("Failed to connect to test server " ++ + lists:flatten(io_lib:format("Host:~p ~n Port:~p~n Error:~p~n", + [Host, Port, Error]))) end end), spawn(fun () -> @@ -476,9 +503,8 @@ cnct2tstsrvr([Host, Port]) when is_list(Host), is_list(Port) -> receive {'DOWN', Mon, process, ConnHandler, Reason} -> receive after 1000 -> ok end, - halt("test server connection handler terminated: " - ++ - lists:flatten(io_lib:format("~p", [Reason]))) + halt("test server connection handler terminated: " ++ + lists:flatten(io_lib:format("~p", [Reason]))) end end). @@ -613,19 +639,34 @@ setup_dist_opts(Config) -> ++ "-ssl_dist_opt server_certfile " ++ SKC ++ " " ++ "-ssl_dist_opt client_certfile " ++ CKC ++ " "; true -> - "-proto_dist inet_tls " - ++ "-ssl_dist_opt server_certfile " ++ SC ++ " " - ++ "-ssl_dist_opt server_keyfile " ++ SK ++ " " - ++ "-ssl_dist_opt server_cacertfile " ++ SCA ++ " " - ++ "-ssl_dist_opt server_verify verify_peer " - ++ "-ssl_dist_opt server_fail_if_no_peer_cert true " - ++ "-ssl_dist_opt server_ciphers DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA " - ++ "-ssl_dist_opt server_dhfile " ++ Dhfile ++ " " - ++ "-ssl_dist_opt client_certfile " ++ CC ++ " " - ++ "-ssl_dist_opt client_keyfile " ++ CK ++ " " - ++ "-ssl_dist_opt client_cacertfile " ++ CCA ++ " " - ++ "-ssl_dist_opt client_verify verify_peer " - ++ "-ssl_dist_opt client_ciphers DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA " + case os:type() of + {win32, _} -> + "-proto_dist inet_tls " + ++ "-ssl_dist_opt server_certfile " ++ SKC ++ " " + ++ "-ssl_dist_opt server_cacertfile " ++ SCA ++ " " + ++ "-ssl_dist_opt server_verify verify_peer " + ++ "-ssl_dist_opt server_fail_if_no_peer_cert true " + ++ "-ssl_dist_opt server_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA " + ++ "-ssl_dist_opt server_dhfile " ++ Dhfile ++ " " + ++ "-ssl_dist_opt client_certfile " ++ CKC ++ " " + ++ "-ssl_dist_opt client_cacertfile " ++ CCA ++ " " + ++ "-ssl_dist_opt client_verify verify_peer " + ++ "-ssl_dist_opt client_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA "; + _ -> + "-proto_dist inet_tls " + ++ "-ssl_dist_opt server_certfile " ++ SC ++ " " + ++ "-ssl_dist_opt server_keyfile " ++ SK ++ " " + ++ "-ssl_dist_opt server_cacertfile " ++ SCA ++ " " + ++ "-ssl_dist_opt server_verify verify_peer " + ++ "-ssl_dist_opt server_fail_if_no_peer_cert true " + ++ "-ssl_dist_opt server_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA " + ++ "-ssl_dist_opt server_dhfile " ++ Dhfile ++ " " + ++ "-ssl_dist_opt client_certfile " ++ CC ++ " " + ++ "-ssl_dist_opt client_keyfile " ++ CK ++ " " + ++ "-ssl_dist_opt client_cacertfile " ++ CCA ++ " " + ++ "-ssl_dist_opt client_verify verify_peer " + ++ "-ssl_dist_opt client_ciphers DHE-RSA-AES256-SHA\:DHE-RSA-AES128-SHA " + end end, MoreOpts = proplists:get_value(additional_dist_opts, Config, []), DistOpts ++ MoreOpts. diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl index 5ea45018e6..8cdfdec2ce 100644 --- a/lib/ssl/test/ssl_session_cache_SUITE.erl +++ b/lib/ssl/test/ssl_session_cache_SUITE.erl @@ -229,7 +229,7 @@ session_cleanup(Config)when is_list(Config) -> check_timer(DelayTimer), - test_server:sleep(?SLEEP), %% Make sure clean has had to run + test_server:sleep(?SLEEP), %% Make sure clean has had time to run undefined = ssl_session_cache:lookup(Cache, {{Hostname, Port}, Id}), undefined = ssl_session_cache:lookup(Cache, {Port, Id}), diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index b7916b96eb..46a8112a41 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -22,6 +22,7 @@ -include("test_server.hrl"). -include("test_server_line.hrl"). +-include_lib("public_key/include/public_key.hrl"). %% Note: This directive should only be used in test suites. -compile(export_all). @@ -673,3 +674,16 @@ cipher_result(Socket, Result) -> session_info_result(Socket) -> ssl:session_info(Socket). + + +public_key(#'PrivateKeyInfo'{privateKeyAlgorithm = + #'PrivateKeyInfo_privateKeyAlgorithm'{algorithm = ?rsaEncryption}, + privateKey = Key}) -> + public_key:der_decode('RSAPrivateKey', iolist_to_binary(Key)); + +public_key(#'PrivateKeyInfo'{privateKeyAlgorithm = + #'PrivateKeyInfo_privateKeyAlgorithm'{algorithm = ?'id-dsa'}, + privateKey = Key}) -> + public_key:der_decode('DSAPrivateKey', iolist_to_binary(Key)); +public_key(Key) -> + Key. diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 64a6a9eaf8..f37baeb9de 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -109,6 +109,9 @@ special_init(TestCase, Config) TestCase == erlang_server_openssl_client_no_wrap_sequence_number -> check_sane_openssl_renegotaite(Config); +special_init(ssl2_erlang_server_openssl_client, Config) -> + check_sane_openssl_sslv2(Config); + special_init(_, Config) -> Config. @@ -168,7 +171,8 @@ all() -> tls1_erlang_server_openssl_client_client_cert, tls1_erlang_server_erlang_client_client_cert, ciphers_rsa_signed_certs, ciphers_dsa_signed_certs, - erlang_client_bad_openssl_server, expired_session, + erlang_client_bad_openssl_server, + expired_session, ssl2_erlang_server_openssl_client]. groups() -> @@ -222,7 +226,6 @@ erlang_client_openssl_server(Config) when is_list(Config) -> %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), - ssl_test_lib:close(Client), process_flag(trap_exit, false), ok. @@ -256,9 +259,9 @@ erlang_server_openssl_client(Config) when is_list(Config) -> port_command(OpenSslPort, Data), ssl_test_lib:check_result(Server, ok), - - ssl_test_lib:close(Server), + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close(Server), close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -306,7 +309,6 @@ tls1_erlang_client_openssl_server_dsa_cert(Config) when is_list(Config) -> %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), - ssl_test_lib:close(Client), process_flag(trap_exit, false), ok. @@ -346,8 +348,8 @@ tls1_erlang_server_openssl_client_dsa_cert(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok), + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), - close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -395,7 +397,6 @@ ssl3_erlang_client_openssl_server_dsa_cert(Config) when is_list(Config) -> %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), - ssl_test_lib:close(Client), process_flag(trap_exit, false), ok. @@ -435,8 +436,8 @@ ssl3_erlang_server_openssl_client_dsa_cert(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok), + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), - close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -475,8 +476,8 @@ erlang_server_openssl_client_reuse_session(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok), + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), - close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -525,7 +526,6 @@ erlang_client_openssl_server_renegotiate(Config) when is_list(Config) -> %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), - ssl_test_lib:close(Client), process_flag(trap_exit, false), ok. @@ -574,7 +574,6 @@ erlang_client_openssl_server_no_wrap_sequence_number(Config) when is_list(Config %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), - ssl_test_lib:close(Client), process_flag(trap_exit, false), ok. @@ -615,8 +614,8 @@ erlang_server_openssl_client_no_wrap_sequence_number(Config) when is_list(Config ssl_test_lib:check_result(Server, ok), + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), - close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -663,7 +662,6 @@ erlang_client_openssl_server_no_server_ca_cert(Config) when is_list(Config) -> %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), - ssl_test_lib:close(Client), process_flag(trap_exit, false), ok. @@ -674,6 +672,7 @@ ssl3_erlang_client_openssl_server(doc) -> ssl3_erlang_client_openssl_server(suite) -> []; ssl3_erlang_client_openssl_server(Config) when is_list(Config) -> + process_flag(trap_exit, true), ServerOpts = ?config(server_opts, Config), ClientOpts = ?config(client_opts, Config), @@ -700,11 +699,11 @@ ssl3_erlang_client_openssl_server(Config) when is_list(Config) -> {options, [{versions, [sslv3]} | ClientOpts]}]), ssl_test_lib:check_result(Client, ok), - - ssl_test_lib:close(Client), - %% Clean close down! + + %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), - test_server:sleep(?SLEEP), + ssl_test_lib:close(Client), + process_flag(trap_exit, false), ok. %%-------------------------------------------------------------------- @@ -714,6 +713,7 @@ ssl3_erlang_server_openssl_client(doc) -> ssl3_erlang_server_openssl_client(suite) -> []; ssl3_erlang_server_openssl_client(Config) when is_list(Config) -> + process_flag(trap_exit, true), ServerOpts = ?config(server_opts, Config), {_, ServerNode, _} = ssl_test_lib:run_where(Config), @@ -734,10 +734,10 @@ ssl3_erlang_server_openssl_client(Config) when is_list(Config) -> OpenSslPort = open_port({spawn, Cmd}, [stderr_to_stdout]), ssl_test_lib:check_result(Server, ok), - - close_port(OpenSslPort), %% openssl server first + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), - test_server:sleep(?SLEEP), + close_port(OpenSslPort), + process_flag(trap_exit, false), ok. %%-------------------------------------------------------------------- @@ -779,7 +779,7 @@ ssl3_erlang_client_openssl_server_client_cert(Config) when is_list(Config) -> ssl_test_lib:check_result(Client, ok), - %% Clean close down! + %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), ssl_test_lib:close(Client), process_flag(trap_exit, false), @@ -824,9 +824,9 @@ ssl3_erlang_server_openssl_client_client_cert(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok), - close_port(OpenSslPort), %% openssl server first + %% Clean close down! Server needs to be closed first !! + close_port(OpenSslPort), ssl_test_lib:close(Server), - %% Clean close down! process_flag(trap_exit, false), ok. @@ -907,10 +907,10 @@ tls1_erlang_client_openssl_server(Config) when is_list(Config) -> [{versions, [tlsv1]} | ClientOpts]}]), ssl_test_lib:check_result(Client, ok), - - ssl_test_lib:close(Client), - %% Clean close down! + + %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), + ssl_test_lib:close(Client), process_flag(trap_exit, false), ok. @@ -943,9 +943,9 @@ tls1_erlang_server_openssl_client(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok), - %% Clean close down! - close_port(OpenSslPort), + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), + close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -989,7 +989,7 @@ tls1_erlang_client_openssl_server_client_cert(Config) when is_list(Config) -> ssl_test_lib:check_result(Client, ok), - %% Clean close down! + %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), ssl_test_lib:close(Client), process_flag(trap_exit, false), @@ -1034,9 +1034,9 @@ tls1_erlang_server_openssl_client_client_cert(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok), - %% Clean close down! - close_port(OpenSslPort), + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), + close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -1071,9 +1071,7 @@ tls1_erlang_server_erlang_client_client_cert(Config) when is_list(Config) -> [{versions, [tlsv1]} | ClientOpts]}]), ssl_test_lib:check_result(Server, ok, Client, ok), - ssl_test_lib:close(Server), - %% Clean close down! process_flag(trap_exit, false), ok. %%-------------------------------------------------------------------- @@ -1136,7 +1134,7 @@ cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> CertFile = proplists:get_value(certfile, ServerOpts), KeyFile = proplists:get_value(keyfile, ServerOpts), - Cmd = "openssl s_server -accept " ++ integer_to_list(Port) ++ + Cmd = "openssl s_server -accept " ++ integer_to_list(Port) ++ version_flag(Version) ++ " -cert " ++ CertFile ++ " -key " ++ KeyFile ++ "", test_server:format("openssl cmd: ~p~n", [Cmd]), @@ -1171,8 +1169,8 @@ cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> Result = ssl_test_lib:wait_for_result(Client, ok), + %% Clean close down! Server needs to be closed first !! close_port(OpenSslPort), - %% Clean close down! ssl_test_lib:close(Client), Return = case Result of @@ -1184,6 +1182,12 @@ cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> process_flag(trap_exit, false), Return. + +version_flag(tlsv1) -> + " -tls1 "; +version_flag(sslv3) -> + " -ssl3 ". + %%-------------------------------------------------------------------- erlang_client_bad_openssl_server(doc) -> [""]; @@ -1199,26 +1203,26 @@ erlang_client_bad_openssl_server(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(node()), CertFile = proplists:get_value(certfile, ServerOpts), KeyFile = proplists:get_value(keyfile, ServerOpts), - + Cmd = "openssl s_server -accept " ++ integer_to_list(Port) ++ - " -cert " ++ CertFile ++ " -key " ++ KeyFile ++ "", - + " -cert " ++ CertFile ++ " -key " ++ KeyFile ++ "", + test_server:format("openssl cmd: ~p~n", [Cmd]), OpensslPort = open_port({spawn, Cmd}, [stderr_to_stdout]), - + wait_for_openssl_server(), Client0 = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {?MODULE, server_sent_garbage, []}}, - {options, - [{versions, [tlsv1]} | ClientOpts]}]), + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, server_sent_garbage, []}}, + {options, + [{versions, [tlsv1]} | ClientOpts]}]), %% Send garbage port_command(OpensslPort, ?OPENSSL_GARBAGE), - + test_server:sleep(?SLEEP), Client0 ! server_sent_garbage, @@ -1228,17 +1232,16 @@ erlang_client_bad_openssl_server(Config) when is_list(Config) -> ssl_test_lib:close(Client0), %% Make sure openssl does not hang and leave zombie process - Client1 = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, no_result_msg, []}}, - {options, - [{versions, [tlsv1]} | ClientOpts]}]), - - ssl_test_lib:close(Client1), - - %% Clean close down! + Client1 = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, no_result_msg, []}}, + {options, + [{versions, [tlsv1]} | ClientOpts]}]), + + %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), + ssl_test_lib:close(Client1), process_flag(trap_exit, false), ok. @@ -1297,6 +1300,7 @@ expired_session(Config) when is_list(Config) -> {mfa, {ssl_test_lib, no_result, []}}, {from, self()}, {options, ClientOpts}]), + %% Clean close down! Server needs to be closed first !! close_port(OpensslPort), ssl_test_lib:close(Client2), process_flag(trap_exit, false). @@ -1329,8 +1333,8 @@ ssl2_erlang_server_openssl_client(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, {error,"protocol version"}), + %% Clean close down! Server needs to be closed first !! ssl_test_lib:close(Server), - close_port(OpenSslPort), process_flag(trap_exit, false), ok. @@ -1433,3 +1437,11 @@ check_sane_openssl_renegotaite(Config) -> _ -> Config end. + +check_sane_openssl_sslv2(Config) -> + case os:cmd("openssl version") of + "OpenSSL 1.0.0e" ++ _ -> + {skip, "Known option bug"}; + _ -> + Config + end. diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 8286201df4..2255798f1d 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 4.1.6 +SSL_VSN = 5.0 diff --git a/lib/stdlib/doc/src/supervisor.xml b/lib/stdlib/doc/src/supervisor.xml index 30514dfee9..cddb55e5c5 100644 --- a/lib/stdlib/doc/src/supervisor.xml +++ b/lib/stdlib/doc/src/supervisor.xml @@ -158,7 +158,7 @@ child_spec() = {Id,StartFunc,Restart,Shutdown,Type,Modules} death causes the temporary process to be terminated) and a <c>transient</c> child process should be restarted only if it terminates abnormally, i.e. with another exit reason - than <c>normal</c>.</p> + than <c>normal</c>, <c>shutdown</c> or <c>{shutdown,Term}</c>.</p> </item> <item> <p><c>Shutdown</c> defines how a child process should be @@ -355,14 +355,23 @@ child_spec() = {Id,StartFunc,Restart,Shutdown,Type,Modules} <desc> <p>Tells the supervisor <c><anno>SupRef</anno></c> to terminate the given child.</p> + <p>If the supervisor is not <c>simple_one_for_one</c>, - <c><anno>Id</anno></c> must be the child specification identifier. The - process, if there is one, is terminated but the child - specification is kept by the supervisor. The child process - may later be restarted by the supervisor. The child process - can also be restarted explicitly by calling + <c><anno>Id</anno></c> must be the child specification + identifier. The process, if there is one, is terminated and, + unless it is a temporary child, the child specification is + kept by the supervisor. The child process may later be + restarted by the supervisor. The child process can also be + restarted explicitly by calling <c>restart_child/2</c>. Use <c>delete_child/2</c> to remove the child specification.</p> + + <p>If the child is temporary, the child specification is deleted as + soon as the process terminates. This means + that <c>delete_child/2</c> has no meaning + and <c>restart_child/2</c> can not be used for these + children.</p> + <p>If the supervisor is <c>simple_one_for_one</c>, <c><anno>Id</anno></c> must be the child process' <c>pid()</c>. I the specified process is alive, but is not a child of the given @@ -399,26 +408,34 @@ child_spec() = {Id,StartFunc,Restart,Shutdown,Type,Modules} <name name="restart_child" arity="2"/> <fsummary>Restart a terminated child process belonging to a supervisor.</fsummary> <desc> - <p>Tells the supervisor <c><anno>SupRef</anno></c> to restart a child process - corresponding to the child specification identified by - <c><anno>Id</anno></c>. The child specification must exist and - the corresponding child process must not be running.</p> - <p>See <seealso marker="#SupRef"><c>start_child/2</c></seealso> for a description of - <c>SupRef</c>.</p> - <p>If the child specification identified by <c><anno>Id</anno></c> does not - exist, the function returns <c>{error,not_found}</c>. If - the child specification exists but the corresponding process - is already running, the function returns + <p>Tells the supervisor <c><anno>SupRef</anno></c> to restart + a child process corresponding to the child specification + identified by <c><anno>Id</anno></c>. The child + specification must exist and the corresponding child process + must not be running.</p> + <p>Note that for temporary children, the child specification + is automatically deleted when the child terminates, and thus + it is not possible to restart such children.</p> + <p>See <seealso marker="#SupRef"><c>start_child/2</c></seealso> + for a description of <c>SupRef</c>.</p> + <p>If the child specification identified + by <c><anno>Id</anno></c> does not exist, the function + returns <c>{error,not_found}</c>. If the child specification + exists but the corresponding process is already running, the + function returns <c>{error,running}</c>.</p> - <p>If the child process start function returns <c>{ok,<anno>Child</anno>}</c> - or <c>{ok,<anno>Child</anno>,<anno>Info</anno>}</c>, the pid is added to the supervisor - and the function returns the same value.</p> + <p>If the child process start function + returns <c>{ok,<anno>Child</anno>}</c> + or <c>{ok,<anno>Child</anno>,<anno>Info</anno>}</c>, the pid + is added to the supervisor and the function returns the same + value.</p> <p>If the child process start function returns <c>ignore</c>, the pid remains set to <c>undefined</c> and the function returns <c>{ok,undefined}</c>.</p> - <p>If the child process start function returns an error tuple or - an erroneous value, or if it fails, the function returns - <c>{error,<anno>Error</anno>}</c> where <c><anno>Error</anno></c> is a term containing + <p>If the child process start function returns an error tuple + or an erroneous value, or if it fails, the function returns + <c>{error,<anno>Error</anno>}</c> + where <c><anno>Error</anno></c> is a term containing information about the error.</p> </desc> </func> diff --git a/lib/stdlib/src/gen.erl b/lib/stdlib/src/gen.erl index 574146b1cd..5d803091b6 100644 --- a/lib/stdlib/src/gen.erl +++ b/lib/stdlib/src/gen.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -273,7 +273,7 @@ reply({To, Tag}, Reply) -> %%%----------------------------------------------------------------- %%% Misc. functions. %%%----------------------------------------------------------------- -where({global, Name}) -> global:safe_whereis_name(Name); +where({global, Name}) -> global:whereis_name(Name); where({local, Name}) -> whereis(Name). name_register({local, Name} = LN) -> diff --git a/lib/stdlib/src/gen_fsm.erl b/lib/stdlib/src/gen_fsm.erl index 3db8c9f4f2..57734a075c 100644 --- a/lib/stdlib/src/gen_fsm.erl +++ b/lib/stdlib/src/gen_fsm.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -296,7 +296,7 @@ get_proc_name({local, Name}) -> exit(process_not_registered) end; get_proc_name({global, Name}) -> - case global:safe_whereis_name(Name) of + case global:whereis_name(Name) of undefined -> exit(process_not_registered_globally); Pid when Pid =:= self() -> @@ -318,7 +318,7 @@ get_parent() -> name_to_pid(Name) -> case whereis(Name) of undefined -> - case global:safe_whereis_name(Name) of + case global:whereis_name(Name) of undefined -> exit(could_not_find_registerd_name); Pid -> @@ -348,12 +348,15 @@ init_it(Starter, Parent, Name0, Mod, Args, Options) -> proc_lib:init_ack(Starter, {ok, self()}), loop(Parent, Name, StateName, StateData, Mod, Timeout, Debug); {stop, Reason} -> + unregister_name(Name0), proc_lib:init_ack(Starter, {error, Reason}), exit(Reason); ignore -> + unregister_name(Name0), proc_lib:init_ack(Starter, ignore), exit(normal); {'EXIT', Reason} -> + unregister_name(Name0), proc_lib:init_ack(Starter, {error, Reason}), exit(Reason); Else -> @@ -366,6 +369,13 @@ name({local,Name}) -> Name; name({global,Name}) -> Name; name(Pid) when is_pid(Pid) -> Pid. +unregister_name({local,Name}) -> + _ = (catch unregister(Name)); +unregister_name({global,Name}) -> + _ = global:unregister_name(Name); +unregister_name(Pid) when is_pid(Pid) -> + Pid. + %%----------------------------------------------------------------- %% The MAIN loop %%----------------------------------------------------------------- diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index dd0ef74f30..6f075bbe5a 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -820,7 +820,7 @@ get_proc_name({local, Name}) -> exit(process_not_registered) end; get_proc_name({global, Name}) -> - case global:safe_whereis_name(Name) of + case global:whereis_name(Name) of undefined -> exit(process_not_registered_globally); Pid when Pid =:= self() -> @@ -842,7 +842,7 @@ get_parent() -> name_to_pid(Name) -> case whereis(Name) of undefined -> - case global:safe_whereis_name(Name) of + case global:whereis_name(Name) of undefined -> exit(could_not_find_registerd_name); Pid -> diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 99bcbd722e..246d535943 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -48,7 +48,7 @@ split(Subject,RE) -> Subject :: iodata() | unicode:charlist(), RE :: mp() | iodata() | unicode:charlist(), Options :: [ Option ], - Option :: anchored | global | notbol | noteol | notempty + Option :: anchored | notbol | noteol | notempty | {offset, non_neg_integer()} | {newline, nl_spec()} | bsr_anycrlf | bsr_unicode | {return, ReturnType} | {parts, NumParts} | group | trim | CompileOpt, diff --git a/lib/stdlib/src/supervisor.erl b/lib/stdlib/src/supervisor.erl index f20ea18fd0..2dd5ccce7a 100644 --- a/lib/stdlib/src/supervisor.erl +++ b/lib/stdlib/src/supervisor.erl @@ -1238,15 +1238,15 @@ report_error(Error, Reason, Child, SupName) -> error_logger:error_report(supervisor_report, ErrorMsg). -extract_child(Child) when is_pid(Child#child.pid) -> - [{pid, Child#child.pid}, +extract_child(Child) when is_list(Child#child.pid) -> + [{nb_children, length(Child#child.pid)}, {name, Child#child.name}, {mfargs, Child#child.mfargs}, {restart_type, Child#child.restart_type}, {shutdown, Child#child.shutdown}, {child_type, Child#child.child_type}]; extract_child(Child) -> - [{nb_children, length(Child#child.pid)}, + [{pid, Child#child.pid}, {name, Child#child.name}, {mfargs, Child#child.mfargs}, {restart_type, Child#child.restart_type}, diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 2f4958760b..0e8849b5b3 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -74,7 +74,7 @@ -export([bad_table/1, types/1]). -export([otp_9423/1]). --export([init_per_testcase/2, end_per_testcase/2]). +-export([init_per_testcase/2]). %% Convenience for manual testing -export([random_test/0]). @@ -176,6 +176,7 @@ groups() -> meta_newdel_unnamed, meta_newdel_named]}]. init_per_suite(Config) -> + erts_debug:set_internal_state(available_internal_state, true), Config. end_per_suite(_Config) -> @@ -304,7 +305,6 @@ t_match_spec_run(Config) when is_list(Config) -> end, repeat_for_permutations(F, N_MS) end, - test_terms(Fun, skip_refc_check), ?line verify_etsmem(EtsMem). @@ -324,7 +324,7 @@ t_match_spec_run_test(List, MS, Result) -> %% Check that tracing agree Self = self(), - {Tracee, MonRef} = spawn_monitor(fun() -> ms_tracee(Self, List) end), + {Tracee, MonRef} = my_spawn_monitor(fun() -> ms_tracee(Self, List) end), receive {Tracee, ready} -> ok end, MST = lists:map(fun(Clause) -> ms_clause_ets_to_trace(Clause) end, MS), @@ -585,7 +585,6 @@ select_fail_do(Opts) -> memory(doc) -> ["Whitebox test of ets:info(X,memory)"]; memory(suite) -> []; memory(Config) when is_list(Config) -> - ?line erts_debug:set_internal_state(available_internal_state, true), ?line ok = chk_normal_tab_struct_size(), repeat_for_opts(memory_do,[compressed]), ?line catch erts_debug:set_internal_state(available_internal_state, false). @@ -1947,7 +1946,7 @@ evil_update_counter(Config) when is_list(Config) -> evil_update_counter_do(Opts) -> ?line EtsMem = etsmem(), ?line process_flag(trap_exit, true), - ?line Pids = [spawn_link(fun() -> evil_counter(I,Opts) end) || I <- lists:seq(1, 40)], + ?line Pids = [my_spawn_link(fun() -> evil_counter(I,Opts) end) || I <- lists:seq(1, 40)], ?line wait_for_all(gb_sets:from_list(Pids)), ?line verify_etsmem(EtsMem), ok. @@ -2153,24 +2152,24 @@ heir_do(Opts) -> Combos), %% No heir - {Founder1,MrefF1} = spawn_monitor(fun()->heir_founder(Master,foo_data,Opts)end), + {Founder1,MrefF1} = my_spawn_monitor(fun()->heir_founder(Master,foo_data,Opts)end), Founder1 ! {go, none}, ?line {"No heir",Founder1} = receive_any(), ?line {'DOWN', MrefF1, process, Founder1, normal} = receive_any(), ?line undefined = ets:info(foo), %% An already dead heir - {Heir2,MrefH2} = spawn_monitor(fun()->die end), + {Heir2,MrefH2} = my_spawn_monitor(fun()->die end), ?line {'DOWN', MrefH2, process, Heir2, normal} = receive_any(), - {Founder2,MrefF2} = spawn_monitor(fun()->heir_founder(Master,foo_data,Opts)end), + {Founder2,MrefF2} = my_spawn_monitor(fun()->heir_founder(Master,foo_data,Opts)end), Founder2 ! {go, Heir2}, ?line {"No heir",Founder2} = receive_any(), ?line {'DOWN', MrefF2, process, Founder2, normal} = receive_any(), ?line undefined = ets:info(foo), %% When heir dies before founder - {Founder3,MrefF3} = spawn_monitor(fun()->heir_founder(Master,"The dying heir",Opts)end), - {Heir3,MrefH3} = spawn_monitor(fun()->heir_heir(Founder3)end), + {Founder3,MrefF3} = my_spawn_monitor(fun()->heir_founder(Master,"The dying heir",Opts)end), + {Heir3,MrefH3} = my_spawn_monitor(fun()->heir_heir(Founder3)end), Founder3 ! {go, Heir3}, ?line {'DOWN', MrefH3, process, Heir3, normal} = receive_any(), Founder3 ! die_please, @@ -2178,14 +2177,12 @@ heir_do(Opts) -> ?line undefined = ets:info(foo), %% When heir dies and pid reused before founder dies - erts_debug:set_internal_state(available_internal_state,true), NextPidIx = erts_debug:get_internal_state(next_pid), - {Founder4,MrefF4} = spawn_monitor(fun()->heir_founder(Master,"The dying heir",Opts)end), - {Heir4,MrefH4} = spawn_monitor(fun()->heir_heir(Founder4)end), + {Founder4,MrefF4} = my_spawn_monitor(fun()->heir_founder(Master,"The dying heir",Opts)end), + {Heir4,MrefH4} = my_spawn_monitor(fun()->heir_heir(Founder4)end), Founder4 ! {go, Heir4}, ?line {'DOWN', MrefH4, process, Heir4, normal} = receive_any(), erts_debug:set_internal_state(next_pid, NextPidIx), - erts_debug:set_internal_state(available_internal_state,false), {Heir4,MrefH4_B} = spawn_monitor_with_pid(Heir4, fun()-> ?line die_please = receive_any() end), Founder4 ! die_please, @@ -2261,9 +2258,9 @@ heir_heir(Founder, Mode) -> heir_1(HeirData,Mode,Opts) -> io:format("test with heir_data = ~p\n", [HeirData]), Master = self(), - ?line Founder = spawn_link(fun() -> heir_founder(Master,HeirData,Opts) end), + ?line Founder = my_spawn_link(fun() -> heir_founder(Master,HeirData,Opts) end), io:format("founder spawned = ~p\n", [Founder]), - ?line {Heir,Mref} = spawn_monitor(fun() -> heir_heir(Founder,Mode) end), + ?line {Heir,Mref} = my_spawn_monitor(fun() -> heir_heir(Founder,Mode) end), io:format("heir spawned = ~p\n", [{Heir,Mref}]), ?line Founder ! {go, Heir}, ?line {'DOWN', Mref, process, Heir, normal} = receive_any(). @@ -2280,7 +2277,7 @@ give_away_do(Opts) -> Parent = self(), %% Give and then give back - ?line {Receiver,Mref} = spawn_monitor(fun()-> give_away_receiver(T,Parent) end), + ?line {Receiver,Mref} = my_spawn_monitor(fun()-> give_away_receiver(T,Parent) end), ?line give_me = receive_any(), ?line true = ets:give_away(T,Receiver,here_you_are), ?line {'EXIT',{badarg,_}} = (catch ets:lookup(T,key)), @@ -2291,7 +2288,7 @@ give_away_do(Opts) -> %% Give and then let receiver keep it ?line true = ets:insert(T,{key,1}), - ?line {Receiver3,Mref3} = spawn_monitor(fun()-> give_away_receiver(T,Parent) end), + ?line {Receiver3,Mref3} = my_spawn_monitor(fun()-> give_away_receiver(T,Parent) end), ?line give_me = receive_any(), ?line true = ets:give_away(T,Receiver3,here_you_are), ?line {'EXIT',{badarg,_}} = (catch ets:lookup(T,key)), @@ -2303,7 +2300,7 @@ give_away_do(Opts) -> ?line T2 = ets_new(foo,[private | Opts]), ?line true = ets:insert(T2,{key,1}), ?line ets:setopts(T2,{heir,self(),"Som en gummiboll..."}), - ?line {Receiver2,Mref2} = spawn_monitor(fun()-> give_away_receiver(T2,Parent) end), + ?line {Receiver2,Mref2} = my_spawn_monitor(fun()-> give_away_receiver(T2,Parent) end), ?line give_me = receive_any(), ?line true = ets:give_away(T2,Receiver2,here_you_are), ?line {'EXIT',{badarg,_}} = (catch ets:lookup(T2,key)), @@ -2318,12 +2315,12 @@ give_away_do(Opts) -> ?line {'EXIT',{badarg,_}} = (catch ets:give_away(T2,"not a pid","To wrong type")), ?line true = ets:delete(T2), - ?line {ReceiverNeg,MrefNeg} = spawn_monitor(fun()-> give_away_receiver(T2,Parent) end), + ?line {ReceiverNeg,MrefNeg} = my_spawn_monitor(fun()-> give_away_receiver(T2,Parent) end), ?line give_me = receive_any(), ?line {'EXIT',{badarg,_}} = (catch ets:give_away(T2,ReceiverNeg,"A deleted table")), ?line T3 = ets_new(foo,[public | Opts]), - spawn_link(fun()-> {'EXIT',{badarg,_}} = (catch ets:give_away(T3,ReceiverNeg,"From non owner")), + my_spawn_link(fun()-> {'EXIT',{badarg,_}} = (catch ets:give_away(T3,ReceiverNeg,"From non owner")), Parent ! done end), ?line done = receive_any(), @@ -2359,7 +2356,7 @@ setopts_do(Opts) -> Self = self(), ?line T = ets_new(foo,[named_table, private | Opts]), ?line none = ets:info(T,heir), - Heir = spawn_link(fun()->heir_heir(Self) end), + Heir = my_spawn_link(fun()->heir_heir(Self) end), ?line ets:setopts(T,{heir,Heir,"Data"}), ?line Heir = ets:info(T,heir), ?line ets:setopts(T,{heir,self(),"Data"}), @@ -2410,14 +2407,14 @@ bad_table(Config) when is_list(Config) -> bad_table_do(Opts, DummyFile) -> Parent = self(), - {Pid,Mref} = spawn_opt(fun()-> ets_new(priv,[private,named_table | Opts]), - Priv = ets_new(priv,[private | Opts]), - ets_new(prot,[protected,named_table | Opts]), - Prot = ets_new(prot,[protected | Opts]), - Parent ! {self(),Priv,Prot}, - die_please = receive_any() - end, - [link, monitor]), + {Pid,Mref} = my_spawn_opt(fun()-> ets_new(priv,[private,named_table | Opts]), + Priv = ets_new(priv,[private | Opts]), + ets_new(prot,[protected,named_table | Opts]), + Prot = ets_new(prot,[protected | Opts]), + Parent ! {self(),Priv,Prot}, + die_please = receive_any() + end, + [link, monitor]), {Pid,Priv,Prot} = receive_any(), MatchSpec = {{key,'_'}, [], ['$$']}, Fun = fun(X,_) -> X end, @@ -3263,7 +3260,7 @@ delete_large_named_table_1(Name, Flags, Data, Fix) -> ?line lists:foreach(fun({K,_}) -> ets:delete(Tab, K) end, Data) end, Parent = self(), - Pid = spawn_link(fun() -> + Pid = my_spawn_link(fun() -> receive {trace,Parent,call,_} -> ets_new(Name, [named_table]) @@ -3625,7 +3622,7 @@ cycle(Tab, L) -> ets:insert(Tab,list_to_tuple(L)), cycle(Tab, tl(L)++[hd(L)]). -dynamic_go() -> spawn_link(fun dynamic_init/0). +dynamic_go() -> my_spawn_link(fun dynamic_init/0). dynamic_init() -> [dyn_lookup(?MODULE) || _ <- lists:seq(1, 10)]. @@ -3852,7 +3849,7 @@ safe_fixtable_do(Opts) -> Self = self(), ?line {{_,_,_},[{Self,1}]} = ets:info(Tab,safe_fixed), %% Test that an unjustified 'unfix' is a no-op. - {Pid,MRef} = spawn_monitor(fun() -> true = ets:safe_fixtable(Tab,false) end), + {Pid,MRef} = my_spawn_monitor(fun() -> true = ets:safe_fixtable(Tab,false) end), {'DOWN', MRef, process, Pid, normal} = receive M -> M end, ?line true = ets:info(Tab,fixed), ?line {{_,_,_},[{Self,1}]} = ets:info(Tab,safe_fixed), @@ -4256,7 +4253,7 @@ do_heavy_concurrent(Opts) -> ?line ok = fill_tab2(Tab, 0, Size), ?line Procs = lists:map( fun (N) -> - spawn_link( + my_spawn_link( fun () -> do_heavy_concurrent_proc(Tab, Size, N) end) @@ -4860,12 +4857,7 @@ otp_7665_act(Tab,Min,Max,DelNr) -> %% Whitebox testing of meta name table hashing. meta_wb(Config) when is_list(Config) -> ?line EtsMem = etsmem(), - ?line erts_debug:set_internal_state(available_internal_state, true), - try - repeat_for_opts(meta_wb_do) - after - erts_debug:set_internal_state(available_internal_state, false) - end, + repeat_for_opts(meta_wb_do), ?line verify_etsmem(EtsMem). @@ -4934,12 +4926,15 @@ colliding_names(Name) -> grow_shrink(Config) when is_list(Config) -> ?line EtsMem = etsmem(), - grow_shrink_0(lists:seq(3071, 5000), EtsMem). + ?line grow_shrink_0(lists:seq(3071, 5000), EtsMem), + ?line verify_etsmem(EtsMem). grow_shrink_0([N|Ns], EtsMem) -> ?line grow_shrink_1(N, [set]), ?line grow_shrink_1(N, [ordered_set]), - ?line verify_etsmem(EtsMem), + %% Verifying ets-memory here takes too long time, since + %% lock-free allocators were introduced... + %% ?line verify_etsmem(EtsMem), grow_shrink_0(Ns, EtsMem); grow_shrink_0([], _) -> ok. @@ -4986,13 +4981,13 @@ grow_pseudo_deleted_do(Type) -> ?line Left = ets:info(T,size), ?line Mult = get_kept_objects(T), filltabstr(T,Mult), - spawn_opt(fun()-> ?line true = ets:info(T,fixed), - Self ! start, - io:format("Starting to filltabstr... ~p\n",[now()]), - filltabstr(T,Mult,Mult+10000), - io:format("Done with filltabstr. ~p\n",[now()]), - Self ! done - end, [link, {scheduler,2}]), + my_spawn_opt(fun()-> ?line true = ets:info(T,fixed), + Self ! start, + io:format("Starting to filltabstr... ~p\n",[now()]), + filltabstr(T,Mult,Mult+10000), + io:format("Done with filltabstr. ~p\n",[now()]), + Self ! done + end, [link, {scheduler,2}]), ?line start = receive_any(), io:format("Unfixing table...~p nitems=~p\n",[now(),ets:info(T,size)]), ?line true = ets:safe_fixtable(T,false), @@ -5026,13 +5021,13 @@ shrink_pseudo_deleted_do(Type) -> [true]}]), ?line Half = ets:info(T,size), ?line Half = get_kept_objects(T), - spawn_opt(fun()-> ?line true = ets:info(T,fixed), - Self ! start, - io:format("Starting to delete... ~p\n",[now()]), - del_one_by_one_set(T,1,Half+1), - io:format("Done with delete. ~p\n",[now()]), - Self ! done - end, [link, {scheduler,2}]), + my_spawn_opt(fun()-> ?line true = ets:info(T,fixed), + Self ! start, + io:format("Starting to delete... ~p\n",[now()]), + del_one_by_one_set(T,1,Half+1), + io:format("Done with delete. ~p\n",[now()]), + Self ! done + end, [link, {scheduler,2}]), ?line start = receive_any(), io:format("Unfixing table...~p nitems=~p\n",[now(),ets:info(T,size)]), ?line true = ets:safe_fixtable(T,false), @@ -5189,24 +5184,24 @@ smp_unfix_fix_do() -> ?line Deleted = get_kept_objects(T), {Child, Mref} = - spawn_opt(fun()-> ?line true = ets:info(T,fixed), - Parent ! start, - io:format("Child waiting for table to be unfixed... now=~p mem=~p\n", - [now(),ets:info(T,memory)]), - repeat_while(fun()-> ets:info(T,fixed) end), - io:format("Table unfixed. Child Fixating! now=~p mem=~p\n", - [now(),ets:info(T,memory)]), - ?line true = ets:safe_fixtable(T,true), - repeat_while(fun(Key) when Key =< NumOfObjs -> - ets:delete(T,Key), {true,Key+1}; - (Key) -> {false,Key} - end, - Deleted), - ?line 0 = ets:info(T,size), - ?line true = get_kept_objects(T) >= Left, - ?line done = receive_any() - end, - [link, monitor, {scheduler,2}]), + my_spawn_opt(fun()-> ?line true = ets:info(T,fixed), + Parent ! start, + io:format("Child waiting for table to be unfixed... now=~p mem=~p\n", + [now(),ets:info(T,memory)]), + repeat_while(fun()-> ets:info(T,fixed) end), + io:format("Table unfixed. Child Fixating! now=~p mem=~p\n", + [now(),ets:info(T,memory)]), + ?line true = ets:safe_fixtable(T,true), + repeat_while(fun(Key) when Key =< NumOfObjs -> + ets:delete(T,Key), {true,Key+1}; + (Key) -> {false,Key} + end, + Deleted), + ?line 0 = ets:info(T,size), + ?line true = get_kept_objects(T) >= Left, + ?line done = receive_any() + end, + [link, monitor, {scheduler,2}]), ?line start = receive_any(), ?line true = ets:info(T,fixed), @@ -5237,11 +5232,11 @@ otp_8166_do(WC) -> Deleted = NumOfObjs div 2, filltabint(T,NumOfObjs), {ReaderPid, ReaderMref} = - spawn_opt(fun()-> otp_8166_reader(T,NumOfObjs) end, - [link, monitor, {scheduler,2}]), + my_spawn_opt(fun()-> otp_8166_reader(T,NumOfObjs) end, + [link, monitor, {scheduler,2}]), {ZombieCrPid, ZombieCrMref} = - spawn_opt(fun()-> otp_8166_zombie_creator(T,Deleted) end, - [link, monitor, {scheduler,3}]), + my_spawn_opt(fun()-> otp_8166_zombie_creator(T,Deleted) end, + [link, monitor, {scheduler,3}]), repeat(fun() -> ZombieCrPid ! {loop, self()}, zombies_created = receive_any(), @@ -5501,7 +5496,7 @@ run_workers_do(InitF,ExecF,FiniF,Laps, Exclude) -> io:format("smp starting ~p workers\n",[NumOfProcs]), Seeds = [{ProcN,random:uniform(9999)} || ProcN <- lists:seq(1,NumOfProcs)], Parent = self(), - Pids = [spawn_link(fun()-> worker(Seed,InitF,ExecF,FiniF,Laps,Parent,NumOfProcs) end) + Pids = [my_spawn_link(fun()-> worker(Seed,InitF,ExecF,FiniF,Laps,Parent,NumOfProcs) end) || Seed <- Seeds], case Laps of infinite -> Pids; @@ -5550,31 +5545,30 @@ my_tab_to_list(_Ts,'$end_of_table', Acc) -> lists:reverse(Acc); my_tab_to_list(Ts,Key, Acc) -> my_tab_to_list(Ts,ets:next(Ts,Key),[ets:lookup(Ts, Key)| Acc]). -wait_for_all_schedulers_online_to_execute() -> - PMs = lists:map(fun (Sched) -> - spawn_opt(fun () -> ok end, - [monitor, {scheduler, Sched}]) - end, - lists:seq(1,erlang:system_info(schedulers_online))), - lists:foreach(fun ({P, M}) -> - receive - {'DOWN', M, process, P, _} -> ok - end - end, - PMs), - ok. + +wait_for_memory_deallocations() -> + try + erts_debug:set_internal_state(wait, deallocations) + catch + error:undef -> + erts_debug:set_internal_state(available_internal_state, true), + wait_for_memory_deallocations() + end. + etsmem() -> - %% Wait until it is guaranteed that all already scheduled - %% deallocations of DbTable structures have completed. - wait_for_all_schedulers_online_to_execute(), + wait_for_memory_deallocations(), AllTabs = lists:map(fun(T) -> {T,ets:info(T,name),ets:info(T,size), ets:info(T,memory),ets:info(T,type)} end, ets:all()), + + EtsAllocInfo = erlang:system_info({allocator,ets_alloc}), + ErlangMemoryEts = try erlang:memory(ets) catch error:notsup -> notsup end, + Mem = - {try erlang:memory(ets) catch error:notsup -> notsup end, - case erlang:system_info({allocator,ets_alloc}) of + {ErlangMemoryEts, + case EtsAllocInfo of false -> undefined; MemInfo -> CS = lists:foldl( @@ -5651,6 +5645,7 @@ spawn_logger(Procs) -> true -> exit(Proc, kill); _ -> ok end, + erlang:display(process_info(Proc)), receive {'DOWN', Mon, _, _, _} -> ok @@ -5686,7 +5681,7 @@ wait_for_test_procs(Kill) -> ets_test_spawn_logger ! {sync_test_procs, Kill, self()}, receive test_procs_synced -> ok end. -log_test_proc(Proc) -> +log_test_proc(Proc) when is_pid(Proc) -> ets_test_spawn_logger ! {new_test_proc, Proc}, Proc. @@ -5698,9 +5693,17 @@ my_spawn_link(Fun) -> log_test_proc(spawn_link(Fun)). my_spawn_link(M,F,A) -> log_test_proc(spawn_link(M,F,A)). %%my_spawn_link(N,M,F,A) -> log_test_proc(spawn_link(N,M,F,A)). -my_spawn_opt(Fun,Opts) -> log_test_proc(spawn_opt(Fun,Opts)). -%%my_spawn_opt(M,F,A,Opts) -> log_test_proc(spawn_opt(M,F,A,Opts)). -%%my_spawn_opt(N,M,F,A,Opts) -> log_test_proc(spawn_opt(N,M,F,A,Opts)). +my_spawn_opt(Fun,Opts) -> + case spawn_opt(Fun,Opts) of + Pid when is_pid(Pid) -> log_test_proc(Pid); + {Pid, _} = Res when is_pid(Pid) -> log_test_proc(Pid), Res + end. + +my_spawn_monitor(Fun) -> + Res = spawn_monitor(Fun), + {Pid, _} = Res, + log_test_proc(Pid), + Res. repeat(_Fun, 0) -> ok; @@ -5763,11 +5766,11 @@ spawn_monitor_with_pid(Pid, Fun, N, M) when N > M*10 -> spawn_monitor_with_pid(Pid, Fun, N, M*10); spawn_monitor_with_pid(Pid, Fun, N, M) -> ?line false = is_process_alive(Pid), - case spawn(fun()-> case self() of - Pid -> Fun(); - _ -> die - end - end) of + case my_spawn(fun()-> case self() of + Pid -> Fun(); + _ -> die + end + end) of Pid -> {Pid, erlang:monitor(process, Pid)}; Other -> diff --git a/lib/stdlib/test/gen_server_SUITE.erl b/lib/stdlib/test/gen_server_SUITE.erl index a614d6595d..7fb8d54f2d 100644 --- a/lib/stdlib/test/gen_server_SUITE.erl +++ b/lib/stdlib/test/gen_server_SUITE.erl @@ -694,7 +694,7 @@ multicall_down(Config) when is_list(Config) -> %% We use 'global' as a gen_server to call. ?line {Good, Bad} = gen_server:multi_call([Name, node()], global_name_server, - {whereis, gurkburk}, + info, 3000), io:format("good = ~p, bad = ~p~n", [Good, Bad]), ?line [Name] = Bad, diff --git a/lib/stdlib/test/supervisor_SUITE.erl b/lib/stdlib/test/supervisor_SUITE.erl index e709cf62ba..d3d140abbc 100644 --- a/lib/stdlib/test/supervisor_SUITE.erl +++ b/lib/stdlib/test/supervisor_SUITE.erl @@ -29,7 +29,8 @@ end_per_testcase/2]). %% Internal export --export([init/1, terminate_all_children/1]). +-export([init/1, terminate_all_children/1, + middle9212/0, gen_server9212/0, handle_info/2]). %% API tests -export([ sup_start_normal/1, sup_start_ignore_init/1, @@ -58,7 +59,8 @@ -export([child_unlink/1, tree/1, count_children_memory/1, do_not_save_start_parameters_for_temporary_children/1, do_not_save_child_specs_for_temporary_children/1, - simple_one_for_one_scale_many_temporary_children/1]). + simple_one_for_one_scale_many_temporary_children/1, + simple_global_supervisor/1]). %%------------------------------------------------------------------------- @@ -77,7 +79,8 @@ all() -> {group, abnormal_termination}, child_unlink, tree, count_children_memory, do_not_save_start_parameters_for_temporary_children, do_not_save_child_specs_for_temporary_children, - simple_one_for_one_scale_many_temporary_children, temporary_bystander]. + simple_one_for_one_scale_many_temporary_children, temporary_bystander, + simple_global_supervisor]. groups() -> [{sup_start, [], @@ -120,7 +123,9 @@ end_per_group(_GroupName, Config) -> init_per_testcase(count_children_memory, Config) -> try erlang:memory() of - _ -> Config + _ -> + erts_debug:set_internal_state(available_internal_state, true), + Config catch error:notsup -> {skip, "+Meamin used during test; erlang:memory/1 not available"} end; @@ -128,6 +133,9 @@ init_per_testcase(_Case, Config) -> erlang:display(_Case), Config. +end_per_testcase(count_children_memory, _Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + ok; end_per_testcase(_Case, _Config) -> ok. @@ -1136,25 +1144,25 @@ count_children_memory(Config) when is_list(Config) -> [supervisor:start_child(sup_test, []) || _Ignore <- lists:seq(1,1000)], garbage_collect(), - _Size1 = erlang:memory(processes_used), + _Size1 = proc_memory(), Children = supervisor:which_children(sup_test), - _Size2 = erlang:memory(processes_used), + _Size2 = proc_memory(), ChildCount = get_child_counts(sup_test), - _Size3 = erlang:memory(processes_used), + _Size3 = proc_memory(), [supervisor:start_child(sup_test, []) || _Ignore2 <- lists:seq(1,1000)], garbage_collect(), Children2 = supervisor:which_children(sup_test), - Size4 = erlang:memory(processes_used), + Size4 = proc_memory(), ChildCount2 = get_child_counts(sup_test), - Size5 = erlang:memory(processes_used), + Size5 = proc_memory(), garbage_collect(), Children3 = supervisor:which_children(sup_test), - Size6 = erlang:memory(processes_used), + Size6 = proc_memory(), ChildCount3 = get_child_counts(sup_test), - Size7 = erlang:memory(processes_used), + Size7 = proc_memory(), 1000 = length(Children), [1,1000,0,1000] = ChildCount, @@ -1180,6 +1188,10 @@ count_children_memory(Config) when is_list(Config) -> [terminate(SupPid, Pid, child, kill) || {undefined, Pid, worker, _Modules} <- Children3], [1,0,0,0] = get_child_counts(sup_test). +proc_memory() -> + erts_debug:set_internal_state(wait, deallocations), + erlang:memory(processes_used). + %%------------------------------------------------------------------------- do_not_save_start_parameters_for_temporary_children(doc) -> ["Temporary children shall not be restarted so they should not " @@ -1379,6 +1391,92 @@ terminate_all_children([]) -> done. +%%------------------------------------------------------------------------- +%% OTP-9212. Restart of global supervisor. +simple_global_supervisor(_Config) -> + kill_supervisor(), + kill_worker(), + exit_worker(), + restart_worker(), + ok. + +kill_supervisor() -> + {Top, Sup2_1, Server_1} = start9212(), + + %% Killing a supervisor isn't really supported, but try it anyway... + exit(Sup2_1, kill), + timer:sleep(200), + Sup2_2 = global:whereis_name(sup2), + Server_2 = global:whereis_name(server), + true = is_pid(Sup2_2), + true = is_pid(Server_2), + true = Sup2_1 =/= Sup2_2, + true = Server_1 =/= Server_2, + + stop9212(Top). + +handle_info({fail, With, After}, _State) -> + timer:sleep(After), + erlang:error(With). + +kill_worker() -> + {Top, _Sup2, Server_1} = start9212(), + exit(Server_1, kill), + timer:sleep(200), + Server_2 = global:whereis_name(server), + true = is_pid(Server_2), + true = Server_1 =/= Server_2, + stop9212(Top). + +exit_worker() -> + %% Very much the same as kill_worker(). + {Top, _Sup2, Server_1} = start9212(), + Server_1 ! {fail, normal, 0}, + timer:sleep(200), + Server_2 = global:whereis_name(server), + true = is_pid(Server_2), + true = Server_1 =/= Server_2, + stop9212(Top). + +restart_worker() -> + {Top, _Sup2, Server_1} = start9212(), + ok = supervisor:terminate_child({global, sup2}, child), + {ok, _Child} = supervisor:restart_child({global, sup2}, child), + Server_2 = global:whereis_name(server), + true = is_pid(Server_2), + true = Server_1 =/= Server_2, + stop9212(Top). + +start9212() -> + Middle = {middle,{?MODULE,middle9212,[]}, permanent,2000,supervisor,[]}, + InitResult = {ok, {{one_for_all,3,60}, [Middle]}}, + {ok, TopPid} = start_link(InitResult), + + Sup2 = global:whereis_name(sup2), + Server = global:whereis_name(server), + true = is_pid(Sup2), + true = is_pid(Server), + {TopPid, Sup2, Server}. + +stop9212(Top) -> + Old = process_flag(trap_exit, true), + exit(Top, kill), + timer:sleep(200), + undefined = global:whereis_name(sup2), + undefined = global:whereis_name(server), + check_exit([Top]), + _ = process_flag(trap_exit, Old), + ok. + +middle9212() -> + Child = {child, {?MODULE,gen_server9212,[]},permanent, 2000, worker, []}, + InitResult = {ok, {{one_for_all,3,60}, [Child]}}, + supervisor:start_link({global,sup2}, ?MODULE, InitResult). + +gen_server9212() -> + InitResult = {ok, []}, + gen_server:start_link({global,server}, ?MODULE, InitResult, []). + %%------------------------------------------------------------------------- terminate(Pid, Reason) when Reason =/= supervisor -> diff --git a/lib/stdlib/test/supervisor_bridge_SUITE.erl b/lib/stdlib/test/supervisor_bridge_SUITE.erl index c4d696564d..b3056ff41a 100644 --- a/lib/stdlib/test/supervisor_bridge_SUITE.erl +++ b/lib/stdlib/test/supervisor_bridge_SUITE.erl @@ -19,8 +19,9 @@ -module(supervisor_bridge_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2,starting/1, - mini_terminate/1,mini_die/1,badstart/1]). --export([client/1,init/1,internal_loop_init/1,terminate/2]). + mini_terminate/1,mini_die/1,badstart/1, + simple_global_supervisor/1]). +-export([client/1,init/1,internal_loop_init/1,terminate/2,server9212/0]). -include_lib("test_server/include/test_server.hrl"). -define(bridge_name,supervisor_bridge_SUITE_server). @@ -31,7 +32,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [starting, mini_terminate, mini_die, badstart]. + [starting, mini_terminate, mini_die, badstart, simple_global_supervisor]. groups() -> []. @@ -138,7 +139,9 @@ init(3) -> receive {InternalPid,init_done} -> {ok,InternalPid,self()} - end. + end; +init({4,Result}) -> + Result. internal_loop_init(Parent) -> register(?work_bridge_name, self()), @@ -160,7 +163,9 @@ terminate(Reason,{Parent,Worker}) -> io:format("Terminating bridge...\n"), exit(Worker,kill), Parent ! {dying,Reason}, - anything. + anything; +terminate(_Reason, _State) -> + any. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -197,3 +202,30 @@ badstart(Config) when is_list(Config) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% OTP-9212. Restart of global supervisor. + +simple_global_supervisor(suite) -> []; +simple_global_supervisor(doc) -> "Globally registered supervisor."; +simple_global_supervisor(Config) when is_list(Config) -> + ?line Dog = test_server:timetrap({seconds,10}), + + Child = {child, {?MODULE,server9212,[]}, permanent, 2000, worker, []}, + InitResult = {ok, {{one_for_all,3,60}, [Child]}}, + {ok, Sup} = + supervisor:start_link({local,bridge9212}, ?MODULE, {4,InitResult}), + + BN_1 = global:whereis_name(?bridge_name), + ?line exit(BN_1, kill), + timer:sleep(200), + BN_2 = global:whereis_name(?bridge_name), + ?line true = is_pid(BN_2), + ?line true = BN_1 =/= BN_2, + + ?line process_flag(trap_exit, true), + exit(Sup, kill), + ?line receive {'EXIT', Sup, killed} -> ok end, + ?line test_server:timetrap_cancel(Dog), + ok. + +server9212() -> + supervisor_bridge:start_link({global,?bridge_name}, ?MODULE, 3). diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl index fb9744d759..e21bd1b88c 100644 --- a/lib/tools/src/cover.erl +++ b/lib/tools/src/cover.erl @@ -522,7 +522,7 @@ call(Request) -> {?SERVER,Reply} -> Reply end, - erlang:demonitor(Ref), + erlang:demonitor(Ref, [flush]), Return end. @@ -545,7 +545,7 @@ remote_call(Node,Request) -> {?SERVER,Reply} -> Reply end, - erlang:demonitor(Ref), + erlang:demonitor(Ref, [flush]), Return end. diff --git a/lib/wx/examples/demo/Makefile b/lib/wx/examples/demo/Makefile index 98d7c6a130..8afa0e780e 100755..100644 --- a/lib/wx/examples/demo/Makefile +++ b/lib/wx/examples/demo/Makefile @@ -80,7 +80,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk docs: -release_spec: +release_spec: opt $(INSTALL_DIR) $(EXRELSYSDIR) $(INSTALL_DATA) $(TESTSRC) $(EXRELSYSDIR) $(INSTALL_DATA) $(TESTTARGETS) $(EXRELSYSDIR) diff --git a/lib/wx/examples/simple/Makefile b/lib/wx/examples/simple/Makefile index 41f0b46eb1..66f5952f0d 100644 --- a/lib/wx/examples/simple/Makefile +++ b/lib/wx/examples/simple/Makefile @@ -51,7 +51,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk docs: -release_spec: +release_spec: opt $(INSTALL_DIR) $(EXRELSYSDIR) $(INSTALL_DATA) $(TESTSRC) $(EXRELSYSDIR) $(INSTALL_DATA) copy.xpm sample.xpm $(TESTTARGETS) $(EXRELSYSDIR) diff --git a/lib/wx/examples/simple/hello.erl b/lib/wx/examples/simple/hello.erl index dc845ddfbb..dc845ddfbb 100755..100644 --- a/lib/wx/examples/simple/hello.erl +++ b/lib/wx/examples/simple/hello.erl diff --git a/lib/wx/examples/simple/menu.erl b/lib/wx/examples/simple/menu.erl index d573fcf13a..d573fcf13a 100755..100644 --- a/lib/wx/examples/simple/menu.erl +++ b/lib/wx/examples/simple/menu.erl diff --git a/lib/wx/examples/simple/minimal.erl b/lib/wx/examples/simple/minimal.erl index dca4adc643..dca4adc643 100755..100644 --- a/lib/wx/examples/simple/minimal.erl +++ b/lib/wx/examples/simple/minimal.erl diff --git a/lib/wx/examples/simple/sample.xpm b/lib/wx/examples/simple/sample.xpm index 3263b15f8a..3263b15f8a 100755..100644 --- a/lib/wx/examples/simple/sample.xpm +++ b/lib/wx/examples/simple/sample.xpm diff --git a/lib/wx/examples/sudoku/Makefile b/lib/wx/examples/sudoku/Makefile index b86c654fdd..33725756b7 100755..100644 --- a/lib/wx/examples/sudoku/Makefile +++ b/lib/wx/examples/sudoku/Makefile @@ -51,7 +51,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk docs: -release_spec: +release_spec: opt $(INSTALL_DIR) $(EXRELSYSDIR) $(INSTALL_DATA) $(TESTSRC) sudoku.hrl $(EXRELSYSDIR) $(INSTALL_DATA) $(TESTTARGETS) $(EXRELSYSDIR) diff --git a/lib/wx/examples/sudoku/sudoku.erl b/lib/wx/examples/sudoku/sudoku.erl index 01caeb9524..01caeb9524 100755..100644 --- a/lib/wx/examples/sudoku/sudoku.erl +++ b/lib/wx/examples/sudoku/sudoku.erl diff --git a/lib/wx/examples/sudoku/sudoku.hrl b/lib/wx/examples/sudoku/sudoku.hrl index 775b563bdc..775b563bdc 100755..100644 --- a/lib/wx/examples/sudoku/sudoku.hrl +++ b/lib/wx/examples/sudoku/sudoku.hrl diff --git a/lib/wx/examples/sudoku/sudoku_board.erl b/lib/wx/examples/sudoku/sudoku_board.erl index 756837582f..756837582f 100755..100644 --- a/lib/wx/examples/sudoku/sudoku_board.erl +++ b/lib/wx/examples/sudoku/sudoku_board.erl diff --git a/lib/wx/examples/sudoku/sudoku_game.erl b/lib/wx/examples/sudoku/sudoku_game.erl index 470aee0e3b..470aee0e3b 100755..100644 --- a/lib/wx/examples/sudoku/sudoku_game.erl +++ b/lib/wx/examples/sudoku/sudoku_game.erl diff --git a/lib/wx/examples/sudoku/sudoku_gui.erl b/lib/wx/examples/sudoku/sudoku_gui.erl index 4aaecfe086..4aaecfe086 100755..100644 --- a/lib/wx/examples/sudoku/sudoku_gui.erl +++ b/lib/wx/examples/sudoku/sudoku_gui.erl diff --git a/lib/wx/examples/xrc/Makefile b/lib/wx/examples/xrc/Makefile index 1dfaae9689..aba58e0d0f 100755..100644 --- a/lib/wx/examples/xrc/Makefile +++ b/lib/wx/examples/xrc/Makefile @@ -28,7 +28,8 @@ TESTMODS = xrc TESTTARGETS = $(TESTMODS:%=%.beam) TESTSRC = $(TESTMODS:%=%.erl) -RESOURCEFILES = appicon.ico basicdlg.xpm custclas.xpm fileopen.gif menu.xrc \ +RESOURCEFILES = \ + appicon.ico basicdlg.xpm custclas.xpm fileopen.gif menu.xrc \ resource.xrc uncenter.xpm variable.xrc appicon.xpm basicdlg.xrc \ custclas.xrc filesave.gif platform.xpm stop.xpm uncenter.xrc \ artprov.xpm controls.xpm derivdlg.xpm frame.xrc platform.xrc \ @@ -59,7 +60,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk docs: -release_spec: +release_spec: opt $(INSTALL_DIR) $(EXRELSYSDIR) $(INSTALL_DATA) $(TESTSRC) $(EXRELSYSDIR) $(INSTALL_DATA) $(TESTTARGETS) $(EXRELSYSDIR) diff --git a/lib/xmerl/doc/examples/test_html.erl b/lib/xmerl/doc/examples/test_html.erl index 3ca15f30f8..3ca15f30f8 100755..100644 --- a/lib/xmerl/doc/examples/test_html.erl +++ b/lib/xmerl/doc/examples/test_html.erl diff --git a/lib/xmerl/doc/examples/xml/test.xml b/lib/xmerl/doc/examples/xml/test.xml index e803a83560..e803a83560 100755..100644 --- a/lib/xmerl/doc/examples/xml/test.xml +++ b/lib/xmerl/doc/examples/xml/test.xml diff --git a/lib/xmerl/doc/examples/xml/test2.xml b/lib/xmerl/doc/examples/xml/test2.xml index 0cb11194fc..0cb11194fc 100755..100644 --- a/lib/xmerl/doc/examples/xml/test2.xml +++ b/lib/xmerl/doc/examples/xml/test2.xml diff --git a/lib/xmerl/doc/examples/xml/test3.xml b/lib/xmerl/doc/examples/xml/test3.xml index dbdc1e62c2..dbdc1e62c2 100755..100644 --- a/lib/xmerl/doc/examples/xml/test3.xml +++ b/lib/xmerl/doc/examples/xml/test3.xml diff --git a/lib/xmerl/doc/examples/xml/test4.xml b/lib/xmerl/doc/examples/xml/test4.xml index e9d85b8d8f..e9d85b8d8f 100755..100644 --- a/lib/xmerl/doc/examples/xml/test4.xml +++ b/lib/xmerl/doc/examples/xml/test4.xml diff --git a/lib/xmerl/doc/examples/xml/test5.xml b/lib/xmerl/doc/examples/xml/test5.xml index e9d85b8d8f..e9d85b8d8f 100755..100644 --- a/lib/xmerl/doc/examples/xml/test5.xml +++ b/lib/xmerl/doc/examples/xml/test5.xml diff --git a/lib/xmerl/doc/examples/xml/testdtd.dtd b/lib/xmerl/doc/examples/xml/testdtd.dtd index 2ce1c513a6..2ce1c513a6 100755..100644 --- a/lib/xmerl/doc/examples/xml/testdtd.dtd +++ b/lib/xmerl/doc/examples/xml/testdtd.dtd diff --git a/lib/xmerl/doc/examples/xml/xmerl.xml b/lib/xmerl/doc/examples/xml/xmerl.xml index f02282dbef..f02282dbef 100755..100644 --- a/lib/xmerl/doc/examples/xml/xmerl.xml +++ b/lib/xmerl/doc/examples/xml/xmerl.xml diff --git a/lib/xmerl/doc/src/part_notes.xml b/lib/xmerl/doc/src/part_notes.xml index 827ffd90e9..827ffd90e9 100755..100644 --- a/lib/xmerl/doc/src/part_notes.xml +++ b/lib/xmerl/doc/src/part_notes.xml diff --git a/lib/xmerl/include/xmerl.hrl b/lib/xmerl/include/xmerl.hrl index 7bb3f4de9b..3760a5cce0 100755..100644 --- a/lib/xmerl/include/xmerl.hrl +++ b/lib/xmerl/include/xmerl.hrl @@ -61,10 +61,11 @@ }). %% namespace node - i.e. a {Prefix, URI} pair -%% TODO: these are not currently used?? /RC -record(xmlNsNode,{ - prefix, - uri = [] + parents = [], % [{atom(),integer()}] + pos, % integer() + prefix, % string() + uri = [] % [] | atom() }). %% XML Element @@ -103,9 +104,10 @@ %% processing instruction -record(xmlPI,{ - name, % atom() - pos, % integer() - value % IOlist() + name, % atom() + parents = [], % [{atom(),integer()}] + pos, % integer() + value % IOlist() }). -record(xmlDocument,{ @@ -154,6 +156,9 @@ declarations = [], % [{Name, Attrs}] doctype_name, doctype_DTD = internal, % internal | DTDId + comments = true, + document = false, + default_attrs = false, rules, keep_rules = false, % delete (ets) tab if false namespace_conformant = false, % true | false diff --git a/lib/xmerl/include/xmerl_xlink.hrl b/lib/xmerl/include/xmerl_xlink.hrl index 375e244c23..375e244c23 100755..100644 --- a/lib/xmerl/include/xmerl_xlink.hrl +++ b/lib/xmerl/include/xmerl_xlink.hrl diff --git a/lib/xmerl/src/xmerl_lib.erl b/lib/xmerl/src/xmerl_lib.erl index 6402f1cbeb..aeb821f411 100644 --- a/lib/xmerl/src/xmerl_lib.erl +++ b/lib/xmerl/src/xmerl_lib.erl @@ -160,8 +160,9 @@ expand_element(E = #xmlText{}, Pos, Parents, Norm) -> E#xmlText{pos = Pos, parents = Parents, value = expand_text(E#xmlText.value, Norm)}; -expand_element(E = #xmlPI{}, Pos, _Parents, Norm) -> +expand_element(E = #xmlPI{}, Pos, Parents, Norm) -> E#xmlPI{pos = Pos, + parents = Parents, value = expand_text(E#xmlPI.value, Norm)}; expand_element(E = #xmlComment{}, Pos, Parents, Norm) -> E#xmlComment{pos = Pos, diff --git a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc index 3b9eaa309c..ec9178ea25 100644 --- a/lib/xmerl/src/xmerl_sax_parser_base.erlsrc +++ b/lib/xmerl/src/xmerl_sax_parser_base.erlsrc @@ -944,14 +944,19 @@ parse_att_value(?STRING_REST("&", Rest), State, Stop, Acc) -> {unparsed, Name, _} -> ?fatal_error(State1, "Unparsed entity reference in attribute value: " ++ Name) end; -parse_att_value(?STRING_UNBOUND_REST(Stop, Rest), State, Stop, Acc) -> +parse_att_value(?STRING_UNBOUND_REST(Stop, Rest), State, Stop, Acc) -> {lists:reverse(Acc), Rest, State}; -parse_att_value(?STRING_UNBOUND_REST($<, _Rest), State, _Stop, _Acc) -> +parse_att_value(?STRING_UNBOUND_REST($<, _Rest), State, _Stop, _Acc) -> ?fatal_error(State, "< not allowed in attribute value"); -parse_att_value(?STRING_UNBOUND_REST(C, Rest), State, Stop, Acc) -> - parse_att_value(Rest, State, Stop, [C|Acc]); -parse_att_value(Bytes, State, Stop, Acc) -> - unicode_incomplete_check([Bytes, State, Stop, Acc, fun parse_att_value/4], +parse_att_value(?STRING_UNBOUND_REST(C, Rest), State, Stop, Acc) -> + if + ?is_char(C) -> + parse_att_value(Rest, State, Stop, [C|Acc]); + true -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character in attribute value: ~p", [C]))) + end; +parse_att_value(Bytes, State, Stop, Acc) -> + unicode_incomplete_check([Bytes, State, Stop, Acc, fun parse_att_value/4], undefined). @@ -1120,10 +1125,10 @@ parse_content(?STRING_UNBOUND_REST(C, Rest), State, Acc, _IgnorableWS) -> ?is_char(C) -> parse_content(Rest, State, [C|Acc], false); true -> - ?fatal_error(State, "Bad character in content: " ++ C) - end; -parse_content(Bytes, State, Acc, IgnorableWS) -> - unicode_incomplete_check([Bytes, State, Acc, IgnorableWS, fun parse_content/4], + ?fatal_error(State, lists:flatten(io_lib:format("Bad character in content: ~p", [C]))) + end; +parse_content(Bytes, State, Acc, IgnorableWS) -> + unicode_incomplete_check([Bytes, State, Acc, IgnorableWS, fun parse_content/4], undefined). @@ -2522,11 +2527,16 @@ parse_entity_value(?STRING_REST("%", Rest), #xmerl_sax_parser_state{file_type=Ty end end; -parse_entity_value(?STRING_UNBOUND_REST(Stop, Rest), State, Stop, Acc) -> +parse_entity_value(?STRING_UNBOUND_REST(Stop, Rest), State, Stop, Acc) -> {lists:reverse(Acc), Rest, State}; -parse_entity_value(?STRING_UNBOUND_REST(C, Rest), State, Stop, Acc) -> - parse_entity_value(Rest, State, Stop, [C|Acc]); -parse_entity_value(Bytes, State, Stop, Acc) -> +parse_entity_value(?STRING_UNBOUND_REST(C, Rest), State, Stop, Acc) -> + if + ?is_char(C) -> + parse_entity_value(Rest, State, Stop, [C|Acc]); + true -> + ?fatal_error(State, lists:flatten(io_lib:format("Bad character in entity value: ~p", [C]))) + end; +parse_entity_value(Bytes, State, Stop, Acc) -> unicode_incomplete_check([Bytes, State, Stop, Acc, fun parse_entity_value/4], undefined). diff --git a/lib/xmerl/src/xmerl_scan.erl b/lib/xmerl/src/xmerl_scan.erl index 25c6547497..ec7ea534d6 100644 --- a/lib/xmerl/src/xmerl_scan.erl +++ b/lib/xmerl/src/xmerl_scan.erl @@ -100,7 +100,21 @@ %% <dd>Set default character set used (default UTF-8). %% This character set is used only if not explicitly given by the XML %% declaration. </dd> +%% <dt><code>{document, Flag}</code></dt> +%% <dd>Set to 'true' if xmerl should return a complete XML document +%% as an xmlDocument record (default 'false').</dd> +%% <dt><code>{comments, Flag}</code></dt> +%% <dd>Set to 'false' if xmerl should skip comments otherwise they will +%% be returned as xmlComment records (default 'true').</dd> +%% <dt><code>{default_attrs, Flag}</code></dt> +%% <dd>Set to 'true' if xmerl should add to elements missing attributes +%% with a defined default value (default 'false').</dd> %% </dl> +%% @type document() = xmlElement() | xmlDocument(). <p> +%% The document returned by <tt>xmerl_scan:string/[1,2]</tt> and +%% <tt>xmerl_scan:file/[1,2]</tt>. The type of the returned record depends on +%% the value of the document option passed to the function. +%% </p> -module(xmerl_scan). @@ -224,7 +238,7 @@ cont_state(X, S=#xmerl_scanner{fun_states = FS}) -> file(F) -> file(F, []). -%% @spec file(Filename::string(), Options::option_list()) -> {xmlElement(),Rest} +%% @spec file(Filename::string(), Options::option_list()) -> {document(),Rest} %% Rest = list() %%% @doc Parse file containing an XML document file(F, Options) -> @@ -264,7 +278,7 @@ int_file_decl(F, Options,_ExtCharset) -> string(Str) -> string(Str, []). -%% @spec string(Text::list(),Options::option_list()) -> {xmlElement(),Rest} +%% @spec string(Text::list(),Options::option_list()) -> {document(),Rest} %% Rest = list() %%% @doc Parse string containing an XML document string(Str, Options) -> @@ -381,6 +395,12 @@ initial_state([{quiet, F}|T], S) when F==true; F==false -> initial_state(T, S#xmerl_scanner{quiet = F}); initial_state([{doctype_DTD,DTD}|T], S) -> initial_state(T,S#xmerl_scanner{doctype_DTD = DTD}); +initial_state([{document, F}|T], S) when is_boolean(F) -> + initial_state(T,S#xmerl_scanner{document = F}); +initial_state([{comments, F}|T], S) when is_boolean(F) -> + initial_state(T,S#xmerl_scanner{comments = F}); +initial_state([{default_attrs, F}|T], S) when is_boolean(F) -> + initial_state(T,S#xmerl_scanner{default_attrs = F}); initial_state([{text_decl,Bool}|T], S) -> initial_state(T,S#xmerl_scanner{text_decl=Bool}); initial_state([{environment,Env}|T], S) -> @@ -518,6 +538,7 @@ scan_document(Str0, S=#xmerl_scanner{event_fun = Event, line = L, col = C, environment=Env, encoding=Charset, + document=Document, validation=ValidateResult}) -> S1 = Event(#xmerl_event{event = started, line = L, @@ -530,8 +551,8 @@ scan_document(Str0, S=#xmerl_scanner{event_fun = Event, Str=if Charset == "utf-8" -> Str0; - Charset=/=undefined -> % Default character set is UTF-8 - xmerl_ucs:to_unicode(Str0,list_to_atom(Charset)); + Charset =/= undefined -> % Default character set is UTF-8 + xmerl_ucs:to_unicode(Str0, list_to_atom(Charset)); true -> %% Charset is undefined if no external input is %% given, and no auto detection of character %% encoding was made. @@ -539,17 +560,17 @@ scan_document(Str0, S=#xmerl_scanner{event_fun = Event, end, %% M1 = erlang:memory(), %% io:format("Memory status before prolog: ~p~n",[M1]), - {T1, S2} = scan_prolog(Str, S1, _StartPos = 1), + {Prolog, Pos, T1, S2} = scan_prolog(Str, S1, _StartPos = 1), %% M2 = erlang:memory(), %% io:format("Memory status after prolog: ~p~n",[M2]), %%io:format("scan_document 2, prolog parsed~n",[]), - T2 = scan_mandatory("<",T1,1,S2,expected_element_start_tag), + T2 = scan_mandatory("<", T1, 1, S2, expected_element_start_tag), %% M3 = erlang:memory(), %% io:format("Memory status before element: ~p~n",[M3]), - {Res, T3, S3} =scan_element(T2,S2,_StartPos = 1), + {Res, T3, S3} = scan_element(T2,S2,Pos), %% M4 = erlang:memory(), %% io:format("Memory status after element: ~p~n",[M4]), - {Tail, S4}=scan_misc(T3, S3, _StartPos = 1), + {Misc, _Pos1, Tail, S4}=scan_misc(T3, S3, Pos + 1), %% M5 = erlang:memory(), %% io:format("Memory status after misc: ~p~n",[M5]), @@ -558,44 +579,52 @@ scan_document(Str0, S=#xmerl_scanner{event_fun = Event, col = S4#xmerl_scanner.col, data = document}, S4), - {Res2,S6} = case validation_mode(ValidateResult) of + {Res2, S6} = case validation_mode(ValidateResult) of off -> - {Res,cleanup(S5)}; + {Res, cleanup(S5)}; dtd when Env == element; Env == prolog -> check_decl2(S5), - case xmerl_validate:validate(S5,Res) of - {'EXIT',{error,Reason}} -> - S5b=cleanup(S5), - ?fatal({failed_validation,Reason}, S5b); - {'EXIT',Reason} -> - S5b=cleanup(S5), - ?fatal({failed_validation,Reason}, S5b); - {error,Reason} -> - S5b=cleanup(S5), - ?fatal({failed_validation,Reason}, S5b); - {error,Reason,_Next} -> - S5b=cleanup(S5), - ?fatal({failed_validation,Reason}, S5b); + case xmerl_validate:validate(S5, Res) of + {'EXIT', {error, Reason}} -> + S5b = cleanup(S5), + ?fatal({failed_validation, Reason}, S5b); + {'EXIT', Reason} -> + S5b = cleanup(S5), + ?fatal({failed_validation, Reason}, S5b); + {error, Reason} -> + S5b = cleanup(S5), + ?fatal({failed_validation, Reason}, S5b); + {error, Reason, _Next} -> + S5b = cleanup(S5), + ?fatal({failed_validation, Reason}, S5b); _XML -> - {Res,cleanup(S5)} + {Res, cleanup(S5)} end; schema -> - case schemaLocations(Res,S5) of - {ok,Schemas} -> + case schemaLocations(Res, S5) of + {ok, Schemas} -> cleanup(S5), %%io:format("Schemas: ~p~nRes: ~p~ninhertih_options(S): ~p~n", %% [Schemas,Res,inherit_options(S5)]), - XSDRes = xmerl_xsd:process_validate(Schemas,Res, + XSDRes = xmerl_xsd:process_validate(Schemas, Res, inherit_options(S5)), - handle_schema_result(XSDRes,S5); + handle_schema_result(XSDRes, S5); _ -> - {Res,cleanup(S5)} + {Res, cleanup(S5)} end; _ -> - {Res,cleanup(S5)} + {Res, cleanup(S5)} end, - {Res2, Tail, S6}. + Res3 = + case Document of + true -> + Content = lists:reverse(Prolog, [Res2 | lists:reverse(Misc)]), + #xmlDocument{content = Content}; + false -> + Res2 + end, + {Res3, Tail, S6}. scan_decl(Str, S=#xmerl_scanner{event_fun = Event, @@ -609,11 +638,11 @@ scan_decl(Str, S=#xmerl_scanner{event_fun = Event, data = document}, S), case scan_prolog(Str, S1, _StartPos = 1) of - {T2="<"++_, S2} -> + {_,_,T2="<"++_, S2} -> {{S2#xmerl_scanner.user_state,T2},[],S2}; - {[], S2}-> + {_,_,[], S2}-> {[],[],S2}; - {T2, S2} -> + {_,_,T2, S2} -> {_,_,S3} = scan_content(T2,S2,[],_Attrs=[],S2#xmerl_scanner.space, _Lang=[],_Parents=[],#xmlNamespace{}), {T2,[],S3} @@ -624,14 +653,17 @@ scan_decl(Str, S=#xmerl_scanner{event_fun = Event, %%% prolog ::= XMLDecl? Misc* (doctypedecl Misc*)? %%% %% empty text declarations are handled by the first function clause. -scan_prolog([], S=#xmerl_scanner{continuation_fun = F}, Pos) -> +scan_prolog(T, S, Pos) -> + scan_prolog(T, S, Pos, []). +scan_prolog([], S=#xmerl_scanner{continuation_fun = F}, Pos, Acc) -> ?dbg("cont()...~n", []), - F(fun(MoreBytes, S1) -> scan_prolog(MoreBytes, S1, Pos) end, - fun(S1) -> {[], S1} end, + F(fun(MoreBytes, S1) -> scan_prolog(MoreBytes, S1, Pos, Acc) end, + fun(S1) -> {Acc, Pos, [], S1} end, S); -scan_prolog("<?xml"++T,S0=#xmerl_scanner{encoding=Charset0,col=Col,line=L},Pos) - when ?whitespace(hd(T)) -> - {Charset,T3, S3}= +scan_prolog("<?xml"++T, + S0=#xmerl_scanner{encoding=Charset0,col=Col,line=L}, + Pos,Acc) when ?whitespace(hd(T)) -> + {Charset, T3, S3} = if Col==1,L==1,S0#xmerl_scanner.text_decl==true -> ?dbg("prolog(\"<?xml\")~n", []), @@ -639,13 +671,13 @@ scan_prolog("<?xml"++T,S0=#xmerl_scanner{encoding=Charset0,col=Col,line=L},Pos) {_,T1,S1} = mandatory_strip(T,S), {Decl,T2, S2}=scan_text_decl(T1,S1), Encoding=Decl#xmlDecl.encoding, - {Encoding,T2, S2#xmerl_scanner{encoding=Encoding}}; + {Encoding, T2, S2#xmerl_scanner{encoding=Encoding}}; Col==1,L==1 -> ?dbg("prolog(\"<?xml\")~n", []), ?bump_col(5), {Decl,T2, S2}=scan_xml_decl(T, S), Encoding=Decl#xmlDecl.encoding, - {Encoding,T2, S2#xmerl_scanner{encoding=Encoding}}; + {Encoding, T2, S2#xmerl_scanner{encoding=Encoding}}; true -> ?fatal({xml_declaration_must_be_first_in_doc,Col,L},S0) end, @@ -659,7 +691,7 @@ scan_prolog("<?xml"++T,S0=#xmerl_scanner{encoding=Charset0,col=Col,line=L},Pos) %% Now transform to declared character set. if Charset==Charset0 -> % Document already transformed to this charset! - scan_prolog(T3, S3, Pos); + scan_prolog(T3, S3, Pos, Acc); Charset0=/=undefined -> %% For example may an external entity %% have the BOM for utf-16 and the internal @@ -668,17 +700,18 @@ scan_prolog("<?xml"++T,S0=#xmerl_scanner{encoding=Charset0,col=Col,line=L},Pos) %% 'iso-10646-utf-1', and Charset will be 'utf-16', all %% legal. %% - scan_prolog(T3,S3#xmerl_scanner{encoding=Charset0},Pos); + scan_prolog(T3,S3#xmerl_scanner{encoding=Charset0},Pos,Acc); Charset == "utf-8" -> - scan_prolog(T3, S3, Pos); + scan_prolog(T3, S3, Pos, Acc); Charset=/=undefined -> % Document not previously transformed T4=xmerl_ucs:to_unicode(T3,list_to_atom(Charset)), - scan_prolog(T4, S3, Pos); + scan_prolog(T4, S3, Pos, Acc); true -> % No encoding info given - scan_prolog(T3, S3, Pos) + scan_prolog(T3, S3, Pos, Acc) end; -scan_prolog("<!DOCTYPE" ++ T, S0=#xmerl_scanner{environment=prolog, - encoding=_Charset}, Pos) -> +scan_prolog("<!DOCTYPE" ++ T, + S0=#xmerl_scanner{environment=prolog,encoding=_Charset}, + Pos, Acc) -> ?dbg("prolog(\"<!DOCTYPE\")~n", []), ?bump_col(9), %% If no known character set assume it is UTF-8 @@ -687,10 +720,13 @@ scan_prolog("<!DOCTYPE" ++ T, S0=#xmerl_scanner{environment=prolog, true -> T end, {T2, S1} = scan_doctype(T1, S), - scan_misc(T2, S1, Pos); -scan_prolog(Str="%"++_T,S=#xmerl_scanner{environment={external,_}},_Pos) -> - scan_ext_subset(Str,S); -scan_prolog(Str, S0 = #xmerl_scanner{user_state=_US,encoding=_Charset},Pos) -> + scan_misc(T2, S1, Pos, Acc); +scan_prolog(Str="%"++_T,S=#xmerl_scanner{environment={external,_}}, + Pos,Acc) -> + {T, S1} = scan_ext_subset(Str,S), + {Acc, Pos, T, S1}; +scan_prolog(Str, S0 = #xmerl_scanner{user_state=_US,encoding=_Charset}, + Pos,Acc) -> ?dbg("prolog(\"<\")~n", []), %% Check for Comments, PI before possible DOCTYPE declaration @@ -700,26 +736,28 @@ scan_prolog(Str, S0 = #xmerl_scanner{user_state=_US,encoding=_Charset},Pos) -> %% Charset==undefined -> xmerl_ucs:to_unicode(Str,'utf-8'); true -> Str end, - {T1, S1}=scan_misc(T, S, Pos), - scan_prolog2(T1,S1,Pos). + {Acc1, Pos1, T1, S1}=scan_misc(T, S, Pos, Acc), + scan_prolog2(T1,S1,Pos1,Acc1). -scan_prolog2([], S=#xmerl_scanner{continuation_fun = F}, Pos) -> +scan_prolog2([], S=#xmerl_scanner{continuation_fun = F}, Pos, Acc) -> ?dbg("cont()...~n", []), - F(fun(MoreBytes, S1) -> scan_prolog2(MoreBytes, S1, Pos) end, - fun(S1) -> {[], S1} end, + F(fun(MoreBytes, S1) -> scan_prolog2(MoreBytes, S1, Pos, Acc) end, + fun(S1) -> {Acc, Pos, [], S1} end, S); -scan_prolog2("<!DOCTYPE" ++ T, S0=#xmerl_scanner{environment=prolog}, Pos) -> +scan_prolog2("<!DOCTYPE" ++ T, S0=#xmerl_scanner{environment=prolog}, + Pos, Acc) -> ?dbg("prolog(\"<!DOCTYPE\")~n", []), ?bump_col(9), {T1, S1} = scan_doctype(T, S), - scan_misc(T1, S1, Pos); -scan_prolog2(Str = "<!" ++ _, S, _Pos) -> + scan_misc(T1, S1, Pos, Acc); +scan_prolog2(Str = "<!" ++ _, S, Pos, Acc) -> ?dbg("prolog(\"<!\")~n", []), %% In e.g. a DTD, we jump directly to markup declarations - scan_ext_subset(Str, S); -scan_prolog2(Str, S0 = #xmerl_scanner{user_state=_US},Pos) -> + {T, S1} = scan_ext_subset(Str, S), + {Acc, Pos, T, S1}; +scan_prolog2(Str, S0 = #xmerl_scanner{user_state=_US},Pos,Acc) -> ?dbg("prolog(\"<\")~n", []), %% Here we consider the DTD provided by doctype_DTD option, @@ -733,7 +771,7 @@ scan_prolog2(Str, S0 = #xmerl_scanner{user_state=_US},Pos) -> end, %% Check for more Comments and PI after DOCTYPE declaration % ?bump_col(1), - scan_misc(Str, S1, Pos). + scan_misc(Str, S1, Pos, Acc). @@ -743,26 +781,46 @@ scan_prolog2(Str, S0 = #xmerl_scanner{user_state=_US},Pos) -> %% - Neither of Comment and PI are returned in the resulting parsed %% structure. %% - scan_misc/3 implements Misc* as that is how the rule is always used -scan_misc([], S=#xmerl_scanner{continuation_fun = F}, Pos) -> +scan_misc(T, S, Pos) -> + scan_misc(T, S, Pos, []). +scan_misc([], S=#xmerl_scanner{continuation_fun = F}, Pos, Acc) -> ?dbg("cont()...~n", []), - F(fun(MoreBytes, S1) -> scan_misc(MoreBytes, S1, Pos) end, - fun(S1) -> {[], S1} end, + F(fun(MoreBytes, S1) -> scan_misc(MoreBytes, S1, Pos, Acc) end, + fun(S1) -> {Acc, Pos, [], S1} end, S); -scan_misc("<!--" ++ T, S0, Pos) -> % Comment +scan_misc("<!--" ++ T, S0=#xmerl_scanner{acc_fun = F, comments=CF}, Pos, Acc) -> % Comment ?bump_col(4), - {_, T1, S1} = scan_comment(T, S, Pos, _Parents = [], _Lang = []), - scan_misc(T1,S1,Pos); -scan_misc("<?" ++ T, S0, Pos) -> % PI + {C, T1, S1} = scan_comment(T, S, Pos, _Parents = [], _Lang = []), + case CF of + true -> + {Acc2, Pos2, S3} = + case F(C, Acc, S1) of + {Acc1, S2} -> + {Acc1, Pos + 1, S2}; + {Acc1, Pos1, S2} -> + {Acc1, Pos1, S2} + end, + scan_misc(T1, S3, Pos2, Acc2); + false -> + scan_misc(T1, S1, Pos, Acc) + end; +scan_misc("<?" ++ T, S0=#xmerl_scanner{acc_fun = F}, Pos, Acc) -> % PI ?dbg("prolog(\"<?\")~n", []), ?bump_col(2), - {_PI, T1, S1} = scan_pi(T, S, Pos), - scan_misc(T1,S1,Pos); -scan_misc(T=[H|_T], S, Pos) when ?whitespace(H) -> + {PI, T1, S1} = scan_pi(T, S, Pos, []), + {Acc2, Pos2, S3} = case F(PI, Acc, S1) of + {Acc1, S2} -> + {Acc1, Pos + 1, S2}; + {Acc1, Pos1, S2} -> + {Acc1, Pos1, S2} + end, + scan_misc(T1,S3,Pos2,Acc2); +scan_misc(T=[H|_T], S, Pos, Acc) when ?whitespace(H) -> ?dbg("prolog(whitespace)~n", []), {_,T1,S1}=strip(T,S), - scan_misc(T1,S1,Pos); -scan_misc(T,S,_Pos) -> - {T,S}. + scan_misc(T1,S1,Pos,Acc); +scan_misc(T,S,Pos,Acc) -> + {Acc,Pos,T,S}. cleanup(S=#xmerl_scanner{keep_rules = false, @@ -789,7 +847,8 @@ scan_xml_decl(T, S) -> Attr = #xmlAttribute{name = version, parents = [{xml, _XMLPos = 1}], value = Vsn}, - scan_xml_decl(T4, S4, #xmlDecl{attributes = [Attr]}). + scan_xml_decl(T4, S4, #xmlDecl{vsn = Vsn, + attributes = [Attr]}). scan_xml_decl([], S=#xmerl_scanner{continuation_fun = F}, Decl) -> ?dbg("cont()...~n", []), @@ -1025,50 +1084,53 @@ xml_vsn([H|T], S=#xmerl_scanner{col = C}, Delim, Acc) -> %%%%%%% [16] PI ::= '<?' PITarget (S (Char* - (Char* '?>' Char*)))? '?>' -scan_pi([], S=#xmerl_scanner{continuation_fun = F}, Pos) -> +scan_pi([], S=#xmerl_scanner{continuation_fun = F}, Pos, Ps) -> ?dbg("cont()...~n", []), - F(fun(MoreBytes, S1) -> scan_pi(MoreBytes, S1, Pos) end, + F(fun(MoreBytes, S1) -> scan_pi(MoreBytes, S1, Pos, Ps) end, fun(S1) -> ?fatal(unexpected_end, S1) end, S); -scan_pi(Str = [H1,H2,H3 | T],S0=#xmerl_scanner{line = L, col = C}, Pos) +scan_pi(Str = [H1,H2,H3 | T],S0=#xmerl_scanner{line = L, col = C}, Pos, Ps) when H1==$x;H1==$X -> %% names beginning with [xX][mM][lL] are reserved for future use. ?bump_col(3), if ((H2==$m) or (H2==$M)) and ((H3==$l) or (H3==$L)) -> - scan_wellknown_pi(T,S,Pos); + scan_wellknown_pi(T,S,Pos,Ps); true -> {Target, _NamespaceInfo, T1, S1} = scan_name(Str, S), - scan_pi(T1, S1, Target, L, C, Pos, []) + scan_pi(T1, S1, Target, L, C, Pos, Ps, []) end; -scan_pi(Str, S=#xmerl_scanner{line = L, col = C}, Pos) -> +scan_pi(Str, S=#xmerl_scanner{line = L, col = C}, Pos, Ps) -> {Target, _NamespaceInfo, T1, S1} = scan_name(Str, S), - scan_pi(T1, S1, Target, L, C, Pos,[]). + scan_pi(T1, S1, Target, L, C, Pos, Ps, []). %%% More info on xml-stylesheet can be found at: %%% "Associating Style Sheets with XML documents", Version 1.0, %%% W3C Recommendation 29 June 1999 (http://www.w3.org/TR/xml-stylesheet/) -scan_wellknown_pi("-stylesheet"++T, S0=#xmerl_scanner{line=L,col=C},Pos) -> +scan_wellknown_pi("-stylesheet"++T, S0=#xmerl_scanner{line=L,col=C},Pos,Ps) -> ?dbg("prolog(\"<?xml-stylesheet\")~n", []), ?bump_col(16), - scan_pi(T, S, "xml-stylesheet",L,C,Pos,[]); -scan_wellknown_pi(Str,S,_Pos) -> + scan_pi(T, S, "xml-stylesheet",L,C,Pos,Ps,[]); +scan_wellknown_pi(Str,S,_Pos,_Ps) -> ?fatal({invalid_target_name, lists:sublist(Str, 1, 10)}, S). -scan_pi([], S=#xmerl_scanner{continuation_fun = F}, Target,L, C, Pos, Acc) -> +scan_pi([], S=#xmerl_scanner{continuation_fun = F}, Target, + L, C, Pos, Ps, Acc) -> ?dbg("cont()...~n", []), - F(fun(MoreBytes, S1) -> scan_pi(MoreBytes, S1, Target, L, C, Pos, Acc) end, + F(fun(MoreBytes, S1) -> scan_pi(MoreBytes, S1, Target, + L, C, Pos, Ps, Acc) end, fun(S1) -> ?fatal(unexpected_end, S1) end, S); scan_pi("?>" ++ T, S0 = #xmerl_scanner{hook_fun = Hook, event_fun = Event}, - Target, L, C, Pos, Acc) -> + Target, L, C, Pos, Ps, Acc) -> ?bump_col(2), PI = #xmlPI{name = Target, + parents = Ps, pos = Pos, value = lists:reverse(Acc)}, S1 = #xmerl_scanner{} = Event(#xmerl_event{event = ended, @@ -1077,22 +1139,25 @@ scan_pi("?>" ++ T, S0 = #xmerl_scanner{hook_fun = Hook, data = PI}, S), {Ret, S2} = Hook(PI, S1), {Ret, T, S2}; -scan_pi([H|T], S, Target, L, C, Pos, Acc) when ?whitespace(H) -> +scan_pi([H|T], S, Target, L, C, Pos, Ps, Acc) when ?whitespace(H) -> ?strip1, - scan_pi2(T1, S1, Target, L, C, Pos, Acc); -scan_pi([H|_T],S,_Target, _L, _C, _Pos, _Acc) -> + scan_pi2(T1, S1, Target, L, C, Pos, Ps, Acc); +scan_pi([H|_T],S,_Target, _L, _C, _Pos, _Ps, _Acc) -> ?fatal({expected_whitespace_OR_end_of_PI,{char,H}}, S). -scan_pi2([], S=#xmerl_scanner{continuation_fun = F}, Target,L, C, Pos, Acc) -> +scan_pi2([], S=#xmerl_scanner{continuation_fun = F}, Target, + L, C, Pos, Ps, Acc) -> ?dbg("cont()...~n", []), - F(fun(MoreBytes, S1) -> scan_pi2(MoreBytes, S1, Target, L, C, Pos, Acc) end, + F(fun(MoreBytes, S1) -> scan_pi2(MoreBytes, S1, Target, + L, C, Pos, Ps, Acc) end, fun(S1) -> ?fatal(unexpected_end, S1) end, S); scan_pi2("?>" ++ T, S0 = #xmerl_scanner{hook_fun = Hook, event_fun = Event}, - Target, L, C, Pos, Acc) -> + Target, L, C, Pos, Ps, Acc) -> ?bump_col(2), PI = #xmlPI{name = Target, + parents = Ps, pos = Pos, value = lists:reverse(Acc)}, S1 = #xmerl_scanner{} = Event(#xmerl_event{event = ended, @@ -1101,10 +1166,10 @@ scan_pi2("?>" ++ T, S0 = #xmerl_scanner{hook_fun = Hook, data = PI}, S), {Ret, S2} = Hook(PI, S1), {Ret, T, S2}; -scan_pi2(Str, S0, Target, L, C, Pos, Acc) -> +scan_pi2(Str, S0, Target, L, C, Pos, Ps, Acc) -> ?bump_col(1), {Ch,T} = wfc_legal_char(Str,S), - scan_pi2(T, S, Target, L, C, Pos, [Ch|Acc]). + scan_pi2(T, S, Target, L, C, Pos, Ps, [Ch|Acc]). @@ -1575,7 +1640,7 @@ scan_markup_decl("<!--" ++ T, S0) -> scan_comment(T, S); scan_markup_decl("<?" ++ T, S0) -> ?bump_col(2), - {_PI, T1, S1} = scan_pi(T, S,_Pos=markup), + {_PI, T1, S1} = scan_pi(T, S,_Pos=markup,[]), strip(T1, S1); scan_markup_decl("<!ELEMENT" ++ T, #xmerl_scanner{rules_read_fun = Read, @@ -1981,7 +2046,7 @@ scan_element(T, S, Pos) -> scan_element(T, S=#xmerl_scanner{line=L,col=C}, Pos, SpaceDefault,Lang, Parents, NS) -> {Name, NamespaceInfo, T1, S1} = scan_name(T, S), - vc_Element_valid(Name,S), + vc_Element_valid(Name,NamespaceInfo,S), ?strip2, scan_element(T2, S2, Pos, Name, L, C, _Attrs = [], Lang, Parents, NamespaceInfo, NS, @@ -2016,7 +2081,8 @@ scan_element("/>" ++ T, S0 = #xmerl_scanner{hook_fun = Hook, Attrs = lists:reverse(Attrs0), E=processed_whole_element(S, Pos, Name, Attrs, Lang, Parents,NSI,Namespace), - wfc_unique_att_spec(Attrs,S), + #xmlElement{attributes = Attrs1} = E, + wfc_unique_att_spec(Attrs1,S), S1 = #xmerl_scanner{} = Event(#xmerl_event{event = ended, line = L, col = C, @@ -2086,9 +2152,10 @@ scan_element(T, S, Pos, Name, StartL, StartC, Attrs, Lang, Parents, P+1 end, Attr = #xmlAttribute{name = AttName, + parents = [{Name, Pos}|Parents], pos = AttrPos, language = Lang, - namespace = NamespaceInfo, + nsinfo = NamespaceInfo, value = AttValue, normalized = IsNorm}, XMLBase=if @@ -2110,6 +2177,14 @@ scan_element(T, S, Pos, Name, StartL, StartC, Attrs, Lang, Parents, scan_element(T4, S5, Pos, Name, StartL, StartC, [Attr|Attrs], Lang, Parents, NSI, NewNS, SpaceDefault). +get_default_attrs(S = #xmerl_scanner{rules_read_fun = Read}, ElemName) -> + case Read(elem_def, ElemName, S) of + #xmlElement{attributes = Attrs} -> + [ {AttName, AttValue} || + {AttName, _, AttValue, _, _} <- Attrs, AttValue =/= no_value ]; + _ -> [] + end. + get_att_type(S=#xmerl_scanner{rules_read_fun=Read},AttName,ElemName) -> case Read(elem_def,ElemName,S) of #xmlElement{attributes = Attrs} -> @@ -2139,6 +2214,23 @@ processed_whole_element(S=#xmerl_scanner{hook_fun = _Hook, Pos, Name, Attrs, Lang, Parents, NSI, Namespace) -> Language = check_language(Attrs, Lang), + AllAttrs = + case S#xmerl_scanner.default_attrs of + true -> + [ #xmlAttribute{name = AttName, + parents = [{Name, Pos} | Parents], + language = Lang, + nsinfo = NSI, + namespace = Namespace, + value = AttValue, + normalized = true} || + {AttName, AttValue} <- get_default_attrs(S, Name), + AttValue =/= no_value, + not lists:keymember(AttName, #xmlAttribute.name, Attrs) ]; + false -> + Attrs + end, + {ExpName, ExpAttrs} = case S#xmerl_scanner.namespace_conformant of true -> @@ -2153,14 +2245,15 @@ processed_whole_element(S=#xmerl_scanner{hook_fun = _Hook, TempNamespace = Namespace#xmlNamespace{default = []}, ExpAttrsX = [A#xmlAttribute{ + namespace=Namespace, expanded_name=expanded_name( A#xmlAttribute.name, - A#xmlAttribute.namespace, + A#xmlAttribute.nsinfo, % NSI, - TempNamespace, S)} || A <- Attrs], + TempNamespace, S)} || A <- AllAttrs], {expanded_name(Name, NSI, Namespace, S), ExpAttrsX}; false -> - {Name, Attrs} + {Name, AllAttrs} end, #xmlElement{name = Name, @@ -2194,10 +2287,32 @@ check_namespace(_, _, _, NS) -> expanded_name(Name, [], #xmlNamespace{default = []}, _S) -> Name; -expanded_name(Name, [], #xmlNamespace{default = URI}, _S) -> - {URI, Name}; -expanded_name(_Name, {"xmlns", Local}, _NS, _S) -> % CHECK THIS /JB - {"xmlns",Local}; +expanded_name(Name, [], #xmlNamespace{default = URI}, S) -> + case URI of + 'http://www.w3.org/XML/1998/namespace' -> + ?fatal(cannot_bind_default_namespace_to_xml_namespace_name, S); + 'http://www.w3.org/2000/xmlns/' -> + ?fatal(cannot_bind_default_namespace_to_xmlns_namespace_name, S); + _ -> + {URI, Name} + end; +expanded_name(Name, N = {"xmlns", Local}, #xmlNamespace{nodes = Ns}, S) -> + {_, Value} = lists:keyfind(Local, 1, Ns), + case Name of + 'xmlns:xml' when Value =/= 'http://www.w3.org/XML/1998/namespace' -> + ?fatal({xml_prefix_cannot_be_redeclared, Value}, S); + 'xmlns:xmlns' -> + ?fatal({xmlns_prefix_cannot_be_declared, Value}, S); + _ -> + case Value of + 'http://www.w3.org/XML/1998/namespace' -> + ?fatal({cannot_bind_prefix_to_xml_namespace, Local}, S); + 'http://www.w3.org/2000/xmlns/' -> + ?fatal({cannot_bind_prefix_to_xmlns_namespace, Local}, S); + _ -> + N + end + end; expanded_name(_Name, {Prefix, Local}, #xmlNamespace{nodes = Ns}, S) -> case lists:keysearch(Prefix, 1, Ns) of {value, {_, URI}} -> @@ -2449,9 +2564,23 @@ scan_content("&" ++ T, S0, Pos, Name, Attrs, Space, Lang, Parents, NS, Acc,[]) - _ -> scan_content(string_to_char_set(S1#xmerl_scanner.encoding,ExpRef)++T1,S1,Pos,Name,Attrs,Space,Lang,Parents,NS,Acc,[]) end; -scan_content("<!--" ++ T, S, Pos, Name, Attrs, Space, Lang, Parents, NS, Acc,[]) -> - {_, T1, S1} = scan_comment(T, S, Pos, Parents, Lang), - scan_content(T1, S1, Pos+1, Name, Attrs, Space, Lang, Parents, NS, Acc,[]); +scan_content("<!--" ++ T, S0=#xmerl_scanner{acc_fun = F, comments=CF}, Pos, Name, Attrs, Space, + Lang, Parents, NS, Acc,[]) -> + ?bump_col(4), + {C, T1, S1} = scan_comment(T, S, Pos, Parents, Lang), + case CF of + true -> + {Acc2, Pos2, S3} = + case F(C, Acc, S1) of + {Acc1, S2} -> + {Acc1, Pos + 1, S2}; + {Acc1, Pos1, S2} -> + {Acc1, Pos1, S2} + end, + scan_content(T1, S3, Pos2, Name, Attrs, Space, Lang, Parents, NS, Acc2,[]); + false -> + scan_content(T1, S1, Pos, Name, Attrs, Space, Lang, Parents, NS, Acc,[]) + end; scan_content("<" ++ T, S0, Pos, Name, Attrs, Space, Lang, Parents, NS, Acc,[]) -> ?bump_col(1), {Markup, T1, S1} = @@ -2508,9 +2637,9 @@ scan_content_markup("![CDATA[" ++ T, S0, Pos, _Name, _Attrs, _Space, _Lang, Parents, _NS) -> ?bump_col(8), scan_cdata(T, S, Pos, Parents); -scan_content_markup("?"++T,S0,Pos,_Name,_Attrs,_Space,_Lang,_Parents,_NS) -> +scan_content_markup("?"++T,S0,Pos,_Name,_Attrs,_Space,_Lang,Parents,_NS) -> ?bump_col(1), - scan_pi(T, S, Pos); + scan_pi(T, S, Pos, Parents); scan_content_markup(T, S, Pos, _Name, _Attrs, Space, Lang, Parents, NS) -> scan_element(T, S, Pos, Space, Lang, Parents, NS). @@ -3259,12 +3388,18 @@ mandatory_delimeter_wfc(T,S) -> wfc_unique_att_spec([],_S) -> ok; -wfc_unique_att_spec([#xmlAttribute{name=N}|Atts],S) -> +wfc_unique_att_spec([#xmlAttribute{name=N,expanded_name=EN}|Atts],S) -> case lists:keymember(N,#xmlAttribute.name,Atts) of true -> ?fatal({error,{unique_att_spec_required,N}},S); _ -> - wfc_unique_att_spec(Atts,S) + case S#xmerl_scanner.namespace_conformant andalso + lists:keymember(EN, #xmlAttribute.expanded_name, Atts) of + true -> + ?fatal({error,{unique_att_spec_required,EN}},S); + _ -> + wfc_unique_att_spec(Atts,S) + end end. wfc_legal_char(Chars,S) when is_list(Chars)-> @@ -3313,6 +3448,11 @@ wfc_Internal_parsed_entity(internal,Value,S) -> wfc_Internal_parsed_entity(_,_,_) -> ok. +vc_Element_valid(_Name, {"xmlns", _}, + S = #xmerl_scanner{namespace_conformant = true}) -> + ?fatal({error,{illegal_element_prefix,xmlns}},S); +vc_Element_valid(Name, _, S) -> + vc_Element_valid(Name, S). vc_Element_valid(_Name,#xmerl_scanner{environment=internal_parsed_entity}) -> ok; @@ -3917,7 +4057,7 @@ schemaLocations(El,#xmerl_scanner{schemaLocation=SL}) -> schemaLocations(#xmlElement{attributes=Atts,xmlbase=_Base}) -> Pred = fun(#xmlAttribute{name=schemaLocation}) -> false; - (#xmlAttribute{namespace={_,"schemaLocation"}}) -> false; + (#xmlAttribute{nsinfo={_,"schemaLocation"}}) -> false; (_) -> true end, case lists:dropwhile(Pred,Atts) of diff --git a/lib/xmerl/src/xmerl_validate.erl b/lib/xmerl/src/xmerl_validate.erl index 893e23ca34..4028fef2b9 100644 --- a/lib/xmerl/src/xmerl_validate.erl +++ b/lib/xmerl/src/xmerl_validate.erl @@ -399,25 +399,28 @@ test_attribute_value(_Rule,Attr,_,_) -> %% +type valid_contents([rule()],[xmlElement()])-> %% [xmlElement() | {error,???}. -valid_contents(Rule,XMLS,Rules,S,WSActionMode)-> - case parse(Rule,XMLS,Rules,WSActionMode,S) of - {XML_N,[]}-> - lists:flatten(XML_N); - {_,[#xmlElement{name=Name}|_T]} -> - exit({error,{element,Name,isnt_comprise_in_the_rule,Rule}}); - {_,[#xmlText{}=Txt|_T]} -> - exit({error,{element,text,Txt,isnt_comprise_in_the_rule,Rule}}); - {error,Reason} -> - {error,Reason}; - {error,Reason,N} -> - {error,Reason,N} +valid_contents(Rule, XMLS, Rules, S, WSActionMode)-> + case parse(Rule, XMLS, Rules, WSActionMode, S) of + {error, Reason} -> + {error, Reason}; + {error, Reason, N} -> + {error, Reason, N}; + {XML_N, Rest} -> %The list may consist of xmlComment{} records + case lists:dropwhile(fun(X) when is_record(X, xmlComment) -> true; (_) -> false end, Rest) of + [] -> + lists:flatten(XML_N); + [#xmlElement{name=Name} |_T] -> + exit({error, {element, Name, isnt_comprise_in_the_rule, Rule}}); + [#xmlText{} = Txt |_T] -> + exit({error, {element, text, Txt, isnt_comprise_in_the_rule, Rule}}) + end end. -parse({'*',SubRule},XMLS,Rules,WSaction,S)-> - star(SubRule,XMLS,Rules,WSaction,[],S); -parse({'+',SubRule},XMLS,Rules,WSaction,S) -> - plus(SubRule,XMLS,Rules,WSaction,S); -parse({choice,CHOICE},XMLS,Rules,WSaction,S)-> +parse({'*', SubRule}, XMLS, Rules, WSaction, S)-> + star(SubRule, XMLS, Rules, WSaction, [], S); +parse({'+',SubRule}, XMLS, Rules, WSaction, S) -> + plus(SubRule, XMLS, Rules, WSaction, S); +parse({choice,CHOICE}, XMLS, Rules, WSaction, S)-> % case XMLS of % [] -> % io:format("~p~n",[{choice,CHOICE,[]}]); @@ -426,47 +429,49 @@ parse({choice,CHOICE},XMLS,Rules,WSaction,S)-> % [#xmlText{value=V}|_] -> % io:format("~p~n",[{choice,CHOICE,{text,V}}]) % end, - choice(CHOICE,XMLS,Rules,WSaction,S); -parse(empty,[],_Rules,_WSaction,_S) -> - {[],[]}; -parse({'?',SubRule},XMLS,Rules,_WSaction,S)-> - question(SubRule,XMLS,Rules,S); -parse({seq,List},XMLS,Rules,WSaction,S) -> - seq(List,XMLS,Rules,WSaction,S); -parse(El_Name,[#xmlElement{name=El_Name}=XML|T],Rules,_WSaction,S) + choice(CHOICE, XMLS, Rules, WSaction, S); +parse(empty, [], _Rules, _WSaction, _S) -> + {[], []}; +parse({'?', SubRule}, XMLS, Rules, _WSaction, S)-> + question(SubRule, XMLS, Rules, S); +parse({seq,List}, XMLS, Rules, WSaction, S) -> + seq(List, XMLS, Rules, WSaction, S); +parse(El_Name, [#xmlElement{name=El_Name} = XML |T], Rules, _WSaction, S) when is_atom(El_Name)-> - case do_validation(read_rules(Rules,El_Name),XML,Rules,S) of - {error,R} -> + case do_validation(read_rules(Rules, El_Name), XML, Rules, S) of + {error, R} -> % {error,R}; exit(R); - {error,R,_N}-> + {error, R, _N}-> % {error,R,N}; exit(R); XML_-> - {[XML_],T} + {[XML_], T} end; -parse(any,Cont,Rules,_WSaction,S) -> - case catch parse_any(Cont,Rules,S) of - Err = {error,_} -> Err; - ValidContents -> {ValidContents,[]} +parse(any, Cont, Rules, _WSaction, S) -> + case catch parse_any(Cont, Rules, S) of + Err = {error, _} -> Err; + ValidContents -> {ValidContents, []} end; -parse(El_Name,[#xmlElement{name=Name}|_T]=S,_Rules,_WSa,_S) when is_atom(El_Name)-> +parse(El_Name, [#xmlElement{name=Name} |_T] = XMLS, _Rules, _WSa, _S) when is_atom(El_Name) -> {error, - {element_seq_not_conform,{wait,El_Name},{is,Name}}, - {{next,S},{act,[]}} }; -parse(_El_Name,[#xmlPI{}=H|T],_Rules,_WSa,_S) -> - {[H],T}; -parse('#PCDATA',XML,_Rules,_WSa,_S)-> + {element_seq_not_conform,{wait, El_Name}, {is, Name}}, + {{next, XMLS}, {act, []}}}; +parse(El_Name, [#xmlComment{} |T], Rules, WSa, S) -> + parse(El_Name, T, Rules, WSa, S); +parse(_El_Name, [#xmlPI{} = H |T], _Rules, _WSa, _S) -> + {[H], T}; +parse('#PCDATA', XMLS, _Rules, _WSa, _S)-> %%% PCDATA it is 0 , 1 or more #xmlText{}. - parse_pcdata(XML); -parse(El_Name,[#xmlText{}|_T]=S,_Rules,_WSa,_S)-> + parse_pcdata(XMLS); +parse(El_Name, [#xmlText{}|_T] = XMLS, _Rules, _WSa, _S)-> {error, - {text_in_place_of,El_Name}, - {{next,S},{act,[]}}}; -parse([],_,_,_,_) -> - {error,no_rule}; -parse(Rule,[],_,_,_) -> - {error,{no_xml_element,Rule}}. + {text_in_place_of, El_Name}, + {{next, XMLS}, {act, []}}}; +parse([], _, _, _, _) -> + {error, no_rule}; +parse(Rule, [], _, _, _) -> + {error, {no_xml_element, Rule}}. parse_any([],_Rules,_S) -> []; @@ -618,11 +623,15 @@ el_name(#xmlElement{name=Name})-> parse_pcdata([#xmlText{}=H|T])-> parse_pcdata(T,[H]); +parse_pcdata([#xmlComment{}|T])-> + parse_pcdata(T,[]); parse_pcdata(H) -> {[],H}. parse_pcdata([#xmlText{}=H|T],Acc)-> parse_pcdata(T,Acc++[H]); +parse_pcdata([#xmlComment{}|T],Acc)-> + parse_pcdata(T,Acc); parse_pcdata(H,Acc) -> {Acc,H}. diff --git a/lib/xmerl/src/xmerl_xpath.erl b/lib/xmerl/src/xmerl_xpath.erl index db3d3ac2d6..b3301f2faf 100644 --- a/lib/xmerl/src/xmerl_xpath.erl +++ b/lib/xmerl/src/xmerl_xpath.erl @@ -41,18 +41,13 @@ % xmerl_xpath_parse:parse(xmerl_xpath_scan:tokens("parent::processing-instruction('foo')")). %% </pre> %% -%% @type docEntity() = +%% @type nodeEntity() = %% xmlElement() %% | xmlAttribute() %% | xmlText() %% | xmlPI() %% | xmlComment() -%% @type nodeEntity() = -%% xmlElement() -%% | xmlAttribute() -%% | xmlText() -%% | xmlPI() -%% | xmlNamespace() +%% | xmlNsNode() %% | xmlDocument() %% @type option_list(). <p>Options allows to customize the behaviour of the %% XPath scanner. @@ -303,6 +298,17 @@ write_node(#xmlNode{pos = Pos, node = #xmlText{value = Txt, parents = Ps}}) -> {text, Pos, Txt, Ps}; +write_node(#xmlNode{pos = Pos, + node = #xmlComment{parents = Ps}}) -> + {comment, Pos, '', Ps}; +write_node(#xmlNode{pos = Pos, + node = #xmlPI{name = Name, + parents = Ps}}) -> + {processing_instruction, Pos, Name, Ps}; +write_node(#xmlNode{pos = Pos, + node = #xmlNsNode{parents = Ps, + prefix = Prefix}}) -> + {namespace, Pos, Prefix, Ps}; write_node(_) -> other. @@ -330,18 +336,16 @@ eval_path(rel, PathExpr, C = #xmlContext{}) -> Context = C#xmlContext{nodeset = NodeSet}, S = #state{context = Context}, path_expr(PathExpr, S); -eval_path(filter, {PathExpr, PredExpr}, C = #xmlContext{}) -> +eval_path(filter, {PathExpr, {pred, Pred}}, C = #xmlContext{}) -> S = #state{context = C}, - S1 = path_expr(PathExpr, S), - pred_expr(PredExpr, S1). + S1 = match_expr(PathExpr, S), + eval_pred(Pred, S1). -eval_primary_expr(FC = {function_call,_,_},S = #state{context = Context}) -> +eval_primary_expr(PrimExpr, S = #state{context = Context}) -> %% NewNodeSet = xmerl_xpath_pred:eval(FC, Context), - NewNodeSet = xmerl_xpath_lib:eval(primary_expr, FC, Context), + NewNodeSet = xmerl_xpath_lib:eval(primary_expr, PrimExpr, Context), NewContext = Context#xmlContext{nodeset = NewNodeSet}, - S#state{context = NewContext}; -eval_primary_expr(PrimExpr,_S) -> - exit({primary_expression,{not_implemented, PrimExpr}}). + S#state{context = NewContext}. %% axis(Axis,NodeTest,Context::xmlContext()) -> xmlContext() @@ -384,8 +388,8 @@ axis1(preceding, Tok, N, Acc, Context) -> match_preceding(Tok, N, Acc, Context); axis1(attribute, Tok, N, Acc, Context) -> match_attribute(Tok, N, Acc, Context); -%axis1(namespace, Tok, N, Acc, Context) -> -% match_namespace(Tok, N, Acc, Context); +axis1(namespace, Tok, N, Acc, Context) -> + match_namespace(Tok, N, Acc, Context); axis1(ancestor_or_self, Tok, N, Acc, Context) -> match_ancestor_or_self(Tok, N, Acc, Context); axis1(descendant_or_self, Tok, N, Acc, Context) -> @@ -627,14 +631,58 @@ node_type(#xmlAttribute{}) -> attribute; node_type(#xmlElement{}) -> element; node_type(#xmlText{}) -> text; node_type(#xmlPI{}) -> processing_instruction; -node_type(#xmlNamespace{}) -> namespace; +node_type(#xmlNsNode{}) -> namespace; +node_type(#xmlComment{}) -> comment; node_type(#xmlDocument{}) -> root_node. %% "The namespace axis contains the namespace nodes of the context node; %% the axis will be empty unless the context node is an element." -%match_namespace(_Tok, _N, _Acc, _Context) -> - %% TODO: IMPLEMENT NAMESPACE AXIS -% erlang:fault(not_yet_implemented). +match_namespace(Tok, N, Acc, Context) -> + case N#xmlNode.type of + element -> + #xmlNode{parents = Ps, node = E} = N, + #xmlElement{name = Name, + namespace = NS, + parents = EPs, + pos = Pos} = E, + #xmlNamespace{default = Default, nodes = NSPairs} = NS, + ThisEPs = [{Name, Pos}|EPs], + ThisPs = [N|Ps], + Acc0 = + case Default of + D when D =:= []; D =:= '' -> + {[], 1}; + URI -> + DefaultNSNode = #xmlNsNode{parents = ThisEPs, + pos = 1, + prefix = [], + uri = URI}, + Node = #xmlNode{type = namespace, + node = DefaultNSNode, + parents = ThisPs}, + {[Node], 2} + end, + {Nodes, _I} = + lists:foldr( + fun ({Prefix, URI}, {AccX, I}) -> + NSNode = #xmlNsNode{parents = ThisEPs, + pos = I, + prefix = Prefix, + uri = URI}, + ThisN = #xmlNode{pos = I, + type = namespace, + node = NSNode, + parents = ThisPs}, + {[ThisN | AccX], I + 1} + end, Acc0, NSPairs), + lists:foldr( + fun (ThisN, AccX) -> + match_self(Tok, ThisN, AccX, Context) + end, Acc, Nodes); + _Other -> + %%[] + Acc + end. update_nodeset(Context = #xmlContext{axis_type = AxisType}, NodeSet) -> @@ -655,8 +703,15 @@ update_nodeset(Context = #xmlContext{axis_type = AxisType}, NodeSet) -> node_test(F, N, Context) when is_function(F) -> F(N, Context); +node_test(_Test, #xmlNode{type=attribute,node=#xmlAttribute{name=xmlns}}, + _Context) -> + false; +node_test(_Test, + #xmlNode{type=attribute,node=#xmlAttribute{nsinfo={"xmlns",_Local}}}, + _Context) -> + false; node_test({wildcard, _}, #xmlNode{type=ElAt}, _Context) - when ElAt==element; ElAt==attribute -> + when ElAt==element; ElAt==attribute; ElAt==namespace -> true; node_test({prefix_test, Prefix}, #xmlNode{node = N}, _Context) -> case N of @@ -720,6 +775,9 @@ node_test({name, {_Tag, Prefix, Local}}, [{_Tag, Prefix, Local}, write_node(NSNodes)]), false end; +node_test({name, {_Tag, [], Local}}, + #xmlNode{node = #xmlNsNode{prefix = Local}}, _Context) -> + true; node_test({node_type, NT}, #xmlNode{node = N}, _Context) -> case {NT, N} of {text, #xmlText{}} -> @@ -728,14 +786,18 @@ node_test({node_type, NT}, #xmlNode{node = N}, _Context) -> true; {attribute, #xmlAttribute{}} -> true; - {namespace, #xmlNamespace{}} -> + {namespace, #xmlNsNode{}} -> + true; + {comment, #xmlComment{}} -> + true; + {processing_instruction, #xmlPI{}} -> true; _ -> false end; -node_test({processing_instruction, {literal, _, Name}}, - #xmlNode{node = {processing_instruction, Name, _Data}}, _Context) -> - true; +node_test({processing_instruction, Name1}, + #xmlNode{node = #xmlPI{name = Name2}}, _Context) -> + Name1 == atom_to_list(Name2); node_test(_Other, _N, _Context) -> %io:format("node_test(~p, ~p) -> false.~n", [_Other, write_node(_N)]), false. diff --git a/lib/xmerl/src/xmerl_xpath_lib.erl b/lib/xmerl/src/xmerl_xpath_lib.erl index cfd0e36667..096f54ec30 100644 --- a/lib/xmerl/src/xmerl_xpath_lib.erl +++ b/lib/xmerl/src/xmerl_xpath_lib.erl @@ -49,5 +49,7 @@ primary_expr({function_call, F, Args}, C) -> %% here, we should look up the function in the context provided %% by the caller, but we haven't figured this out yet. exit({not_a_core_function, F}) - end. + end; +primary_expr(PrimExpr, _C) -> + exit({primary_expression, {not_implemented, PrimExpr}}). diff --git a/lib/xmerl/src/xmerl_xpath_parse.yrl b/lib/xmerl/src/xmerl_xpath_parse.yrl index 37576b9e61..f60cea0a2e 100644 --- a/lib/xmerl/src/xmerl_xpath_parse.yrl +++ b/lib/xmerl/src/xmerl_xpath_parse.yrl @@ -144,6 +144,7 @@ Expect 2. %% [7] 'NodeTest' -> 'NameTest' : '$1' . 'NodeTest' -> 'node_type' '(' ')' : {node_type, value('$1')} . +'NodeTest' -> 'processing-instruction' '(' ')' : {node_type, value('$1')} . 'NodeTest' -> 'processing-instruction' '(' 'literal' ')' : {processing_instruction, value('$3')} . diff --git a/lib/xmerl/src/xmerl_xpath_pred.erl b/lib/xmerl/src/xmerl_xpath_pred.erl index 451a09bee3..855b8599fe 100644 --- a/lib/xmerl/src/xmerl_xpath_pred.erl +++ b/lib/xmerl/src/xmerl_xpath_pred.erl @@ -337,6 +337,9 @@ local_name1([#xmlNode{type=element,node=El}|_]) -> local_name1([#xmlNode{type=attribute,node=Att}|_]) -> #xmlAttribute{name=Name,nsinfo=NSI} = Att, local_name2(Name,NSI); +local_name1([#xmlNode{type=namespace,node=N}|_]) -> + #xmlNsNode{prefix=Prefix} = N, + ?string(Prefix); local_name1([#xmlElement{name = Name, nsinfo = NSI}|_]) -> local_name2(Name,NSI). local_name2(Name, NSI) -> @@ -431,6 +434,9 @@ string_value(N=#xmlObj{}) -> string_value(A=#xmlNode{type=attribute}) -> #xmlAttribute{value=AttVal}=A#xmlNode.node, ?string(AttVal); +string_value(N=#xmlNode{type=namespace}) -> + #xmlNsNode{uri=URI}=N#xmlNode.node, + ?string(atom_to_list(URI)); string_value(El=#xmlNode{type=element}) -> #xmlElement{content=C} = El#xmlNode.node, TextValue = fun(#xmlText{value=T},_Fun) -> T; @@ -442,6 +448,9 @@ string_value(El=#xmlNode{type=element}) -> string_value(T=#xmlNode{type=text}) -> #xmlText{value=Txt} = T#xmlNode.node, ?string(Txt); +string_value(T=#xmlNode{type=comment}) -> + #xmlComment{value=Txt} = T#xmlNode.node, + ?string(Txt); string_value(infinity) -> ?string("Infinity"); string_value(neg_infinity) -> ?string("-Infinity"); string_value(A) when is_atom(A) -> diff --git a/lib/xmerl/src/xmerl_xsd.erl b/lib/xmerl/src/xmerl_xsd.erl index dfdc6138ef..ed0890f0d0 100644 --- a/lib/xmerl/src/xmerl_xsd.erl +++ b/lib/xmerl/src/xmerl_xsd.erl @@ -245,21 +245,27 @@ process_validate2({SE,_},Schema,Xml,Opts) -> S4 = validation_options(S3,Opts), validate3(Schema,Xml,S4). -validate3(Schema,Xml,S=#xsd_state{errors=[]}) -> - Ret = {_,S2} = - case catch validate_xml(Xml,S) of - {[XML2],[],Sx} -> - {XML2,Sx}; - {XML2,[],Sx} -> - {XML2,Sx}; - {_,UnValidated,Sx} -> - {Xml,acc_errs(Sx,{error_path(UnValidated,Xml#xmlElement.name),?MODULE, - {unvalidated_rest,UnValidated}})}; - _Err = {error,Reason} -> - {Xml,acc_errs(S,Reason)}; - {'EXIT',Reason} -> - {Xml,acc_errs(S,{error_path(Xml,Xml#xmlElement.name),?MODULE, - {undefined,{internal_error,Reason}}})} +validate3(Schema, Xml,S =#xsd_state{errors=[]}) -> + Ret = {_, S2} = + case catch validate_xml(Xml, S) of + _Err = {error, Reason} -> + {Xml, acc_errs(S, Reason)}; + {'EXIT', Reason} -> + {Xml, acc_errs(S, {error_path(Xml, Xml#xmlElement.name), ?MODULE, + {undefined, {internal_error, Reason}}})}; + {XML2, Rest, Sx} -> + case lists:dropwhile(fun(X) when is_record(X, xmlComment) -> true; (_) -> false end, Rest) of + [] -> + case XML2 of + [XML3] -> + {XML3,Sx}; + XML3 -> + {XML3,Sx} + end; + UnValidated -> + {Xml,acc_errs(Sx,{error_path(UnValidated,Xml#xmlElement.name),?MODULE, + {unvalidated_rest,UnValidated}})} + end end, save_to_file(S2,filename:rootname(Schema)++".tab2"), case S2#xsd_state.errors of @@ -1950,7 +1956,7 @@ fetch_external_schema(Path,S) when is_list(Path) -> {EXSD,S#xsd_state{schema_name=File}} end; {_,{string,String},_} -> %% this is for a user defined fetch fun that returns an xml document on string format. - ?debug("scanning string: ~p~n",[File]), + ?debug("scanning string: ~p~n",[String]), case xmerl_scan:string(String,S#xsd_state.xml_options) of {error,Reason} -> {error,acc_errs(S,{[],?MODULE,{parsing_external_schema_failed,Path,Reason}})}; @@ -2520,9 +2526,9 @@ check_element_type([],#schema_complex_type{name=_Name,block=_Bl,content=C}, {error,{error_path(Checked,undefined),?MODULE, {empty_content_not_allowed,C}}} end; -check_element_type(C,{anyType,_},_Env,_Block,S,_Checked) -> +check_element_type(C, {anyType, _}, _Env, _Block, S, _Checked) -> %% permitt anything - {C,[],S}; + {lists:reverse(C), [], S}; check_element_type(XML=[#xmlText{}|_],Type=#schema_simple_type{}, _Env,_Block,S,_Checked) -> @@ -2585,7 +2591,7 @@ check_element_type(XML=[XMLEl=#xmlElement{name=Name}|RestXML], S6 = check_form(ElName,Name,XMLEl, actual_form_value(CMEl#schema_element.form, S5#xsd_state.elementFormDefault), - S5), + S5), %Step into content of XML element. {Content,_,S7} = case @@ -2605,12 +2611,12 @@ check_element_type(XML=[XMLEl=#xmlElement{name=Name}|RestXML], RestXML, set_scope(S5#xsd_state.scope,set_num_el(S7,S6))}; true -> - {error,{error_path(XMLEl,Name),?MODULE, - {element_not_suitable_with_schema,ElName,S}}}; + {error,{error_path(XMLEl, Name), ?MODULE, + {element_not_suitable_with_schema, ElName, S}}}; _ when S#xsd_state.num_el >= Min -> %% it may be a match error or an optional element not %% present - {[],XML,S#xsd_state{num_el=0}}; + {[], XML, S#xsd_state{num_el=0}}; _ -> {error,{error_path(XMLEl,Name),?MODULE, {element_not_suitable_with_schema,ElName,CMName,CMEl,S}}} @@ -2645,7 +2651,7 @@ check_element_type(XML=[#xmlElement{}|_Rest], check_element_type(XML=[E=#xmlElement{name=Name}|Rest], Any={any,{Namespace,_Occ={Min,_},ProcessorContents}},Env, _Block,S,_Checked) -> - ?debug("check any: {any,{~p,~p,~p}}~n",[Namespace,Occ,ProcessorContents]), + ?debug("check any: {any,{~p,~p,~p}}~n",[Namespace,_Occ,ProcessorContents]), %% ProcessorContents any of lax | strict | skip %% lax: may validate if schema is found %% strict: must validate @@ -2710,8 +2716,11 @@ check_element_type([],CM,_Env,_Block,S,Checked) -> {error,{error_path(Checked,undefined),?MODULE, {empty_content_not_allowed,CM}}} end; +check_element_type([C = #xmlComment{} |Rest],CM,Env,Block,S,Checked) -> + check_element_type(Rest,CM,Env,Block,S,[C |Checked]); check_element_type(XML,CM,_Env,_Block,S,_Checked) -> {error,{error_path(XML,undefined),?MODULE,{match_failure,XML,CM,S}}}. + %% single xml content object and single schema object check_text_type(XML=[#xmlText{}|_],optional_text,S) -> % {XMLTxt,optional_text}; @@ -2730,7 +2739,7 @@ check_text_type([XMLTxt=#xmlText{}|_],CMEl,_S) -> {cannot_contain_text,XMLTxt,CMEl}}}. split_xmlText(XML) -> - splitwith(fun(#xmlText{}) -> true;(_) -> false end,XML). + splitwith(fun(#xmlText{}) -> true;(#xmlComment{}) -> true;(_) -> false end,XML). %% Sequence check_sequence([T=#xmlText{}|Rest],Els,Occ,Env,S,Checked) -> @@ -2773,6 +2782,8 @@ check_sequence(Seq=[_InstEl=#xmlElement{}|_],[El|Els],Occ={_Min,_Max},Env,S,Chec count_num_el(set_num_el(S3,S2)), Ret++Checked) end; +check_sequence([C = #xmlComment{} |Rest], Els, Occ, Env, S, Checked) -> + check_sequence(Rest,Els,Occ,Env,S,[C |Checked]); check_sequence(Rest,[],_Occ,_Env,S,Checked) -> {Checked,Rest,set_num_el(S,0)}; check_sequence([],Els,_Occ,_Env,S,Checked) -> @@ -2869,6 +2880,8 @@ check_all(XML=[E=#xmlElement{name=Name}|RestXML],CM,Occ,Env,S, {element_not_in_all,ElName,E,CM}}, check_all(RestXML,CM,Occ,Env,acc_errs(S,Err),[E|Checked],PrevXML) end; +check_all([C=#xmlComment{} |RestXML], CM, Occ, Env, S, Checked, XML) -> + check_all(RestXML, CM, Occ, Env, S, [C |Checked], XML); check_all(XML,[],_,_,S,Checked,_) -> {Checked,XML,S}; check_all([],CM,_Occ,_,S,Checked,_PrevXML) -> @@ -2920,7 +2933,7 @@ check_target_namespace(XMLEl,S) -> schemaLocations(El=#xmlElement{attributes=Atts},S) -> Pred = fun(#xmlAttribute{name=schemaLocation}) -> false; - (#xmlAttribute{namespace={_,"schemaLocation"}}) -> false; + (#xmlAttribute{nsinfo={_,"schemaLocation"}}) -> false; (_) -> true end, case lists:dropwhile(Pred,Atts) of diff --git a/lib/xmerl/test/xmerl_SUITE.erl b/lib/xmerl/test/xmerl_SUITE.erl index 94c38d4d48..55b6d1844c 100644 --- a/lib/xmerl/test/xmerl_SUITE.erl +++ b/lib/xmerl/test/xmerl_SUITE.erl @@ -58,7 +58,7 @@ groups() -> {ticket_tests, [], [ticket_5998, ticket_7211, ticket_7214, ticket_7430, ticket_6873, ticket_7496, ticket_8156, ticket_8697, - ticket_9411, ticket_9457]}, + ticket_9411, ticket_9457, ticket_9664_schema, ticket_9664_dtd]}, {app_test, [], [{xmerl_app_test, all}]}, {appup_test, [], [{xmerl_appup_test, all}]}]. @@ -284,7 +284,7 @@ export(Config) -> ?line {E,_} = xmerl_scan:file(TestFile), ?line Exported = xmerl:export([E],xmerl_xml,[{prolog,Prolog}]), B = list_to_binary(Exported++"\n"), - ?line {ok,B} = file:read_file(TestFile), + ?line {ok, B} = file:read_file(TestFile), ok. %%---------------------------------------------------------------------- @@ -609,6 +609,38 @@ ticket_9457_cont(Continue, Exception, GlobalState) -> Exception(GlobalState) end. + +ticket_9664_schema(suite) -> []; +ticket_9664_schema(doc) -> + ["Test that comments are handled correct whith"]; +ticket_9664_schema(Config) -> + + ?line {E, _} = xmerl_scan:file(filename:join([?config(data_dir, Config), misc, + "ticket_9664_schema.xml"]),[]), + ?line {ok, S} = xmerl_xsd:process_schema(filename:join([?config(data_dir, Config), misc, + "motorcycles.xsd"])), + ?line {E1, _} = xmerl_xsd:validate(E, S), + + ?line {E1,_} = xmerl_xsd:process_validate(filename:join([?config(data_dir,Config), misc, + "motorcycles.xsd"]),E,[]), + + ?line {E1,_} = xmerl_scan:file(filename:join([?config(data_dir,Config), misc, + "ticket_9664_schema.xml"]), + [{schemaLocation, [{"mc", "motorcycles.xsd"}]}, + {validation, schema}]), + ok. + +ticket_9664_dtd(suite) -> []; +ticket_9664_dtd(doc) -> + ["Test that comments are handled correct whith"]; +ticket_9664_dtd(Config) -> + ?line {E, _} = xmerl_scan:file(filename:join([?config(data_dir, Config), misc, + "ticket_9664_dtd.xml"]),[]), + ?line {E, _} = xmerl_scan:file(filename:join([?config(data_dir, Config), misc, + "ticket_9664_dtd.xml"]),[{validation, true}]), + ok. + + %%====================================================================== %% Support Functions %%====================================================================== diff --git a/lib/xmerl/test/xmerl_SUITE_data/misc.tar.gz b/lib/xmerl/test/xmerl_SUITE_data/misc.tar.gz Binary files differindex fef7431845..ffc1d327a5 100644 --- a/lib/xmerl/test/xmerl_SUITE_data/misc.tar.gz +++ b/lib/xmerl/test/xmerl_SUITE_data/misc.tar.gz diff --git a/lib/xmerl/test/xmerl_SUITE_data/xpath/xpath_abbrev.erl b/lib/xmerl/test/xmerl_SUITE_data/xpath/xpath_abbrev.erl index 850b7f8135..7b6f1e95b3 100644 --- a/lib/xmerl/test/xmerl_SUITE_data/xpath/xpath_abbrev.erl +++ b/lib/xmerl/test/xmerl_SUITE_data/xpath/xpath_abbrev.erl @@ -210,7 +210,7 @@ ticket_7496() -> ?line {Doc3,_} = xmerl_scan:file("documentRoot.xml"), ?line ok = Test(Doc3,"//child",[child,child,child]), ?line ok = Test(Doc3,"//child[@name='beta']",[child]), - ?line [{xmlAttribute,id,[],[],[],[],1,[],"2",false}] = + ?line [{xmlAttribute,id,[],[],[],_,1,[],"2",false}] = xmerl_xpath:string("/documentRoot/parent/child[@name='beta']/@id",Doc3), ?line ok = Test(Doc3,"/documentRoot/parent/child|/documentRoot/parent/pet", [child,child,child,pet,pet]), diff --git a/lib/xmerl/test/xmerl_xsd_SUITE_data/mim.xsd b/lib/xmerl/test/xmerl_xsd_SUITE_data/mim.xsd index 057344cde8..057344cde8 100755..100644 --- a/lib/xmerl/test/xmerl_xsd_SUITE_data/mim.xsd +++ b/lib/xmerl/test/xmerl_xsd_SUITE_data/mim.xsd diff --git a/lib/xmerl/vsn.mk b/lib/xmerl/vsn.mk index 82df8fdeef..de47e3418b 100644 --- a/lib/xmerl/vsn.mk +++ b/lib/xmerl/vsn.mk @@ -1 +1 @@ -XMERL_VSN = 1.2.10 +XMERL_VSN = 1.3 diff --git a/lib/xmerl/xmerl.pub b/lib/xmerl/xmerl.pub index 29a81bbde2..29a81bbde2 100755..100644 --- a/lib/xmerl/xmerl.pub +++ b/lib/xmerl/xmerl.pub |
