aboutsummaryrefslogtreecommitdiffstats
path: root/lib/stdlib/doc/src/unicode.xml
diff options
context:
space:
mode:
authorDan Gudmundsson <[email protected]>2017-04-24 12:41:10 +0200
committerDan Gudmundsson <[email protected]>2017-04-24 12:41:10 +0200
commit739bca3fc267c55d84c8f5c193d16c0b2a7eee13 (patch)
treea7ac2a4c90bdeadd2a313e7d49cecf84d3e1ba28 /lib/stdlib/doc/src/unicode.xml
parent515dc2d603d449ed3621d96028ba403aef05ea7f (diff)
parenta9d0d119837fb0bc52d2bb3d48a47568de9100b4 (diff)
downloadotp-739bca3fc267c55d84c8f5c193d16c0b2a7eee13.tar.gz
otp-739bca3fc267c55d84c8f5c193d16c0b2a7eee13.tar.bz2
otp-739bca3fc267c55d84c8f5c193d16c0b2a7eee13.zip
Merge branch 'dgud/stdlib/unicode-string/OTP-10289'
* dgud/stdlib/unicode-string/OTP-10289: Handle chardata in string:to_float and string:to_list New unicode aware string module that works with unicode:chardata() Add nf(k)d, nf(k)c conversion functions to unicode module Reorder code and whitespace fixes Add unicode_util
Diffstat (limited to 'lib/stdlib/doc/src/unicode.xml')
-rw-r--r--lib/stdlib/doc/src/unicode.xml179
1 files changed, 177 insertions, 2 deletions
diff --git a/lib/stdlib/doc/src/unicode.xml b/lib/stdlib/doc/src/unicode.xml
index 93d0d37456..382b253ba1 100644
--- a/lib/stdlib/doc/src/unicode.xml
+++ b/lib/stdlib/doc/src/unicode.xml
@@ -50,8 +50,35 @@
external entities where this is required. When working inside the
Erlang/OTP environment, it is recommended to keep binaries in UTF-8 when
representing Unicode characters. ISO Latin-1 encoding is supported both
- for backward compatibility and for communication
- with external entities not supporting Unicode character sets.</p>
+ for backward compatibility and for communication
+ with external entities not supporting Unicode character sets.</p>
+ <p>Programs should always operate on a normalized form and compare
+ canonical-equivalent Unicode characters as equal. All characters
+ should thus be normalized to one form once on the system borders.
+ One of the following functions can convert characters to their
+ normalized forms <seealso marker="#characters_to_nfc_list/1">
+ <c>characters_to_nfc_list/1</c></seealso>,
+ <seealso marker="#characters_to_nfc_binary/1">
+ <c>characters_to_nfc_binary/1</c></seealso>,
+ <seealso marker="#characters_to_nfd_list/1">
+ <c>characters_to_nfd_list/1</c></seealso> or
+ <seealso marker="#characters_to_nfd_binary/1">
+ <c>characters_to_nfd_binary/1</c></seealso>.
+ For general text
+ <seealso marker="#characters_to_nfc_list/1">
+ <c>characters_to_nfc_list/1</c></seealso> or
+ <seealso marker="#characters_to_nfc_binary/1">
+ <c>characters_to_nfc_binary/1</c></seealso> is preferred, and
+ for identifiers one of the compatibility normalization
+ functions, such as
+ <seealso marker="#characters_to_nfkc_list/1">
+ <c>characters_to_nfkc_list/1</c></seealso>,
+ is preferred for security reasons.
+ The normalization functions where introduced in OTP 20.
+ Additional information on normalization can be found in the
+ <url href="http://unicode.org/faq/normalization.html">Unicode FAQ</url>.
+ </p>
+
</description>
<datatypes>
@@ -335,6 +362,154 @@ decode_data(Data) ->
</func>
<func>
+ <name name="characters_to_nfc_list" arity="1"/>
+ <fsummary>Normalize characters to a list of canonical equivalent
+ composed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of canonical equivalent Composed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.
+ </p>
+ <p>The result is a list of characters.</p>
+ <code>
+3> unicode:characters_to_nfc_list([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776]]).
+"abc..åäö"
+</code>
+ </desc>
+ </func>
+
+ <func>
+ <name name="characters_to_nfc_binary" arity="1"/>
+ <fsummary>Normalize characters to a utf8 binary of canonical equivalent
+ composed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of canonical equivalent Composed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.</p>
+ <p>The result is an utf8 encoded binary.</p>
+ <code>
+4> unicode:characters_to_nfc_binary([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776]]).
+&lt;&lt;"abc..åäö"/utf8>>
+</code>
+ </desc>
+ </func>
+
+ <func>
+ <name name="characters_to_nfd_list" arity="1"/>
+ <fsummary>Normalize characters to a list of canonical equivalent
+ decomposed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of canonical equivalent Decomposed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.
+ </p>
+ <p>The result is a list of characters.</p>
+ <code>
+1> unicode:characters_to_nfd_list("abc..åäö").
+[97,98,99,46,46,97,778,97,776,111,776]
+</code>
+ </desc>
+ </func>
+
+ <func>
+ <name name="characters_to_nfd_binary" arity="1"/>
+ <fsummary>Normalize characters to a utf8 binary of canonical equivalent
+ decomposed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of canonical equivalent Decomposed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.</p>
+ <p>The result is an utf8 encoded binary.</p>
+ <code>
+2> unicode:characters_to_nfd_binary("abc..åäö").
+&lt;&lt;97,98,99,46,46,97,204,138,97,204,136,111,204,136>>
+</code>
+ </desc>
+ </func>
+
+ <func>
+ <name name="characters_to_nfkc_list" arity="1"/>
+ <fsummary>Normalize characters to a list of canonical equivalent
+ composed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of compatibly equivalent Composed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.
+ </p>
+ <p>The result is a list of characters.</p>
+ <code>
+3> unicode:characters_to_nfkc_list([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776],[65299,65298]]).
+"abc..åäö32"
+</code>
+ </desc>
+ </func>
+
+ <func>
+ <name name="characters_to_nfkc_binary" arity="1"/>
+ <fsummary>Normalize characters to a utf8 binary of compatibly equivalent
+ composed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of compatibly equivalent Composed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.</p>
+ <p>The result is an utf8 encoded binary.</p>
+ <code>
+4> unicode:characters_to_nfkc_binary([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776],[65299,65298]]).
+&lt;&lt;"abc..åäö32"/utf8>>
+</code>
+ </desc>
+ </func>
+
+ <func>
+ <name name="characters_to_nfkd_list" arity="1"/>
+ <fsummary>Normalize characters to a list of compatibly equivalent
+ decomposed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of compatibly equivalent Decomposed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.
+ </p>
+ <p>The result is a list of characters.</p>
+ <code>
+1> unicode:characters_to_nfkd_list(["abc..åäö",[65299,65298]]).
+[97,98,99,46,46,97,778,97,776,111,776,51,50]
+</code>
+ </desc>
+ </func>
+
+ <func>
+ <name name="characters_to_nfkd_binary" arity="1"/>
+ <fsummary>Normalize characters to a utf8 binary of compatibly equivalent
+ decomposed Unicode characters.</fsummary>
+ <desc>
+ <p>Converts a possibly deep list of characters and binaries
+ into a Normalized Form of compatibly equivalent Decomposed
+ characters according to the Unicode standard.</p>
+ <p>Any binaries in the input must be encoded with utf8
+ encoding.</p>
+ <p>The result is an utf8 encoded binary.</p>
+ <code>
+2> unicode:characters_to_nfkd_binary(["abc..åäö",[65299,65298]]).
+&lt;&lt;97,98,99,46,46,97,204,138,97,204,136,111,204,136,51,50>>
+</code>
+ </desc>
+ </func>
+
+ <func>
<name name="encoding_to_bom" arity="1"/>
<fsummary>Create a binary UTF byte order mark from encoding.</fsummary>
<type_desc variable="Bin">