aboutsummaryrefslogtreecommitdiffstats
path: root/lib/stdlib/doc/src/unicode.xml
blob: 245100656459898e3b0bb431d2998fa7c5fdbccf (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
<?xml version="1.0" encoding="utf-8" ?>
<!DOCTYPE erlref SYSTEM "erlref.dtd">

<erlref>
  <header>
    <copyright>
      <year>1996</year>
      <year>2017</year>
      <holder>Ericsson AB, All Rights Reserved</holder>
    </copyright>
    <legalnotice>
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
 
      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

  The Initial Developer of the Original Code is Ericsson AB.
    </legalnotice>

    <title>unicode</title>
    <prepared></prepared>
    <docno></docno>
    <date></date>
    <rev></rev>
  </header>
  <module>unicode</module>
  <modulesummary>Functions for converting Unicode characters.</modulesummary>
  <description>
  <p>This module contains functions for converting between different character 
    representations. It converts between ISO Latin-1 characters and Unicode
    characters, but it can also convert between different Unicode encodings
    (like UTF-8, UTF-16, and UTF-32).</p>
  <p>The default Unicode encoding in Erlang is in binaries UTF-8, which is also
    the format in which built-in functions and libraries in OTP expect to find
    binary Unicode data. In lists, Unicode data is encoded as integers, each
    integer representing one character and encoded simply as the Unicode code
    point for the character.</p> 
  <p>Other Unicode encodings than integers representing code points or UTF-8
    in binaries are referred to as &quot;external encodings&quot;. The ISO
    Latin-1 encoding
    is in binaries and lists referred to as latin1-encoding.</p>
  <p>It is recommended to only use external encodings for communication with
    external entities where this is required. When working inside the
    Erlang/OTP environment, it is recommended to keep binaries in UTF-8 when
    representing Unicode characters. ISO Latin-1 encoding is supported both
    for backward compatibility and for communication
  with external entities not supporting Unicode character sets.</p>
  <p>Programs should always operate on a normalized form and compare
  canonical-equivalent Unicode characters as equal. All characters
  should thus be normalized to one form once on the system borders.
  One of the following functions can convert characters to their
  normalized forms <seealso marker="#characters_to_nfc_list/1">
  <c>characters_to_nfc_list/1</c></seealso>,
  <seealso marker="#characters_to_nfc_binary/1">
    <c>characters_to_nfc_binary/1</c></seealso>,
    <seealso marker="#characters_to_nfd_list/1">
    <c>characters_to_nfd_list/1</c></seealso> or
    <seealso marker="#characters_to_nfd_binary/1">
      <c>characters_to_nfd_binary/1</c></seealso>.
  For general text
  <seealso marker="#characters_to_nfc_list/1">
    <c>characters_to_nfc_list/1</c></seealso> or
    <seealso marker="#characters_to_nfc_binary/1">
      <c>characters_to_nfc_binary/1</c></seealso> is preferred, and
      for identifiers one of the compatibility normalization
      functions, such as
      <seealso marker="#characters_to_nfkc_list/1">
      <c>characters_to_nfkc_list/1</c></seealso>,
      is preferred for security reasons.
      The normalization functions where introduced in OTP 20.
      Additional information on normalization can be found in the
      <url href="http://unicode.org/faq/normalization.html">Unicode FAQ</url>.
  </p>

  </description>

  <datatypes>
    <datatype>
      <name name="encoding"/>
    </datatype>
    <datatype>
      <name name="endian"/>
    </datatype>
    <datatype>
      <name name="unicode_binary"/>
      <desc>
        <p>A <c>binary()</c> with characters encoded in the UTF-8 coding
          standard.</p>
      </desc>
    </datatype>
    <datatype>
      <name name="chardata"/>
    </datatype>
    <datatype>
      <name name="charlist"/>
    </datatype>
    <datatype>
      <name name="external_unicode_binary"/>
      <desc>
        <p>A <c>binary()</c> with characters coded in a user-specified Unicode
          encoding other than UTF-8 (that is, UTF-16 or UTF-32).</p>
      </desc>
    </datatype>
    <datatype>
      <name name="external_chardata"/>
    </datatype>
    <datatype>
      <name name="external_charlist"/>
    </datatype>
    <datatype>
      <name name="latin1_binary"/>
      <desc><p>A <c>binary()</c> with characters coded in ISO Latin-1.</p>
      </desc>
    </datatype>
    <datatype>
      <name name="latin1_char"/>
      <desc><p>An <c>integer()</c> representing a valid ISO Latin-1
         character (0-255).</p>
      </desc>
    </datatype>
    <datatype>
      <name name="latin1_chardata"/>
      <desc><p>Same as <c>iodata()</c>.</p>
      </desc>
    </datatype>
    <datatype>
      <name name="latin1_charlist"/>
      <desc><p>Same as <c>iolist()</c>.</p>
      </desc>
    </datatype>
  </datatypes>

  <funcs>
    <func>
      <name name="bom_to_encoding" arity="1"/>
      <fsummary>Identify UTF byte order marks in a binary.</fsummary>
      <type name="endian"/>
      <type_desc variable="Bin">
        A <c>binary()</c> such that <c>byte_size(<anno>Bin</anno>) &gt;= 4</c>.
      </type_desc>
      <desc>
        <p>Checks for a UTF Byte Order Mark (BOM) in the beginning of a
          binary. If the supplied binary <c><anno>Bin</anno></c> begins with a
          valid BOM for either UTF-8, UTF-16, or UTF-32, the function
          returns the encoding identified along with the BOM length
          in bytes.</p>
        <p>If no BOM is found, the function returns <c>{latin1,0}</c>.</p>
      </desc>
    </func>

    <func>
     <name name="characters_to_binary" arity="1"/>
      <fsummary>Convert a collection of characters to a UTF-8 binary.</fsummary>
      <desc>
        <p>Same as <c>characters_to_binary(<anno>Data</anno>, unicode,
          unicode)</c>.</p>
      </desc>
    </func>

    <func>    
     <name name="characters_to_binary" arity="2"/>
      <fsummary>Convert a collection of characters to a UTF-8 binary.</fsummary>
      <desc>
        <p>Same as <c>characters_to_binary(<anno>Data</anno>, 
          <anno>InEncoding</anno>, unicode)</c>.</p>
      </desc>
    </func>

    <func>
      <name name="characters_to_binary" arity="3"/>
      <fsummary>Convert a collection of characters to a UTF-8 binary.</fsummary>
      <desc>
        <p>Behaves as <seealso marker="#characters_to_list/2">
          <c>characters_to_list/2</c></seealso>, but produces a binary
          instead of a Unicode list.</p>
        <p><c><anno>InEncoding</anno></c> defines how input is to be interpreted
          if binaries are present in <c>Data</c></p>
        <p><c><anno>OutEncoding</anno></c> defines in what format output is to
          be generated.</p>
        <p>Options:</p>
        <taglist>
          <tag><c>unicode</c></tag>
          <item>
            <p>An alias for <c>utf8</c>, as this is the preferred encoding for 
              Unicode characters in binaries.</p>
          </item>
          <tag><c>utf16</c></tag>
          <item>
            <p>An alias for <c>{utf16,big}</c>.</p>
          </item>
          <tag><c>utf32</c></tag>
          <item>
            <p>An alias for <c>{utf32,big}</c>.</p>
          </item>
        </taglist>
        <p>The atoms <c>big</c> and <c>little</c> denote big- or little-endian
          encoding.</p>
        <p>Errors and exceptions occur as in
          <seealso marker="#characters_to_list/2">
          <c>characters_to_list/2</c></seealso>, but the second element
          in tuple <c>error</c> or <c>incomplete</c> is a <c>binary()</c>
          and not a <c>list()</c>.</p>
      </desc>
    </func>

    <func>
      <name name="characters_to_list" arity="1"/>
      <fsummary>Convert a collection of characters to a list of Unicode
        characters.</fsummary>
      <desc>
        <p>Same as <c>characters_to_list(<anno>Data</anno>, unicode)</c>.</p>
      </desc>
    </func>

    <func>
      <name name="characters_to_list" arity="2"/>
      <fsummary>Convert a collection of characters to a list of Unicode
        characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of integers and
          binaries into a list of integers representing Unicode
          characters. The binaries in the input can have characters
          encoded as one of the following:</p>
        <list type="bulleted">
          <item>
            <p>ISO Latin-1 (0-255, one character per byte). Here,
              case parameter <c><anno>InEncoding</anno></c> is to be specified
              as <c>latin1</c>.</p>
          </item>
          <item>
            <p>One of the UTF-encodings, which is specified as parameter 
              <c><anno>InEncoding</anno></c>.</p>
          </item>
        </list>
	<p>
	  Note that integers in the list always represent code points
	  regardless of <c><anno>InEncoding</anno></c> passed. If
	  <c><anno>InEncoding</anno> latin1</c> is passed, only code
	  points &lt; 256 are allowed; otherwise, all valid unicode code
	  points are allowed.
	</p>
        <p>If <c><anno>InEncoding</anno></c> is <c>latin1</c>, parameter
          <c><anno>Data</anno></c> corresponds to the <c>iodata()</c> type,
          but for <c>unicode</c>, parameter <c><anno>Data</anno></c> can
          contain integers &gt; 255
          (Unicode characters beyond the ISO Latin-1 range), which
          makes it invalid as <c>iodata()</c>.</p>
        <p>The purpose of the function is mainly to convert
          combinations of Unicode characters into a pure Unicode
          string in list representation for further processing. For
          writing the data to an external entity, the reverse function
          <seealso marker="#characters_to_binary/3">
          <c>characters_to_binary/3</c></seealso>
          comes in handy.</p>
        <p>Option <c>unicode</c> is an alias for <c>utf8</c>, as this is the
          preferred encoding for Unicode characters in
          binaries. <c>utf16</c> is an alias for <c>{utf16,big}</c> and
          <c>utf32</c> is an alias for <c>{utf32,big}</c>. The atoms <c>big</c>
          and <c>little</c> denote big- or little-endian encoding.</p> 
        <p>If the data cannot be converted, either
          because of illegal Unicode/ISO Latin-1 characters in the list, 
          or because of invalid UTF encoding in any binaries, an error
          tuple is returned. The error tuple contains the tag
          <c>error</c>, a list representing the characters that could be
          converted before the error occurred and a representation of the
          characters including and after the offending integer/bytes. The
          last part is mostly for debugging, as it still constitutes a
          possibly deep or mixed list, or both, not necessarily of the same
          depth as the original data. The error occurs when traversing the
          list and whatever is left to decode is returned "as is".</p>
        <p>However, if the input <c><anno>Data</anno></c> is a pure binary,
          the third part of the error tuple is guaranteed to be a binary as
          well.</p>
        <p>Errors occur for the following reasons:</p>
        <list type="bulleted">
          <item>
            <p>Integers out of range.</p>
            <p>If <c><anno>InEncoding</anno></c> is <c>latin1</c>, 
              an error occurs whenever an integer &gt; 255 is found
              in the lists.</p>
            <p>If <c><anno>InEncoding</anno></c> is of a Unicode type,
              an error occurs whenever either of the following is found:</p>
            <list type="bulleted">
              <item>
                <p>An integer &gt; 16#10FFFF
                  (the maximum Unicode character)</p>
              </item>
	      <item>
                <p>An integer in the range 16#D800 to 16#DFFF (invalid range
                  reserved for UTF-16 surrogate pairs)</p>
              </item>
            </list>
	  </item>
          <item>
            <p>Incorrect UTF encoding.</p>
            <p>If <c><anno>InEncoding</anno></c> is one of the UTF types,
              the bytes in any binaries must be valid in that encoding.</p>
            <p>Errors can occur for various reasons, including the
              following:</p>
            <list type="bulleted">
              <item>
                <p>&quot;Pure&quot; decoding errors 
	          (like the upper bits of the bytes being wrong).</p>
              </item>
              <item>
                <p>The bytes are decoded to a too large number.</p>
              </item> 
              <item>
                <p>The bytes are decoded to a code point in the invalid
	          Unicode range.</p>
              </item>
              <item>
                <p>Encoding is &quot;overlong&quot;, meaning that a number
                  should have been encoded in fewer bytes.</p>
              </item>
            </list>
            <p>The case of a truncated UTF is handled specially, see the
              paragraph about incomplete binaries below.</p>
            <p>If <c><anno>InEncoding</anno></c> is <c>latin1</c>, binaries are
              always valid as long as they contain whole bytes,
              as each byte falls into the valid ISO Latin-1 range.</p>
          </item>
        </list>
        <p>A special type of error is when no actual invalid integers or
          bytes are found, but a trailing <c>binary()</c> consists of too
          few bytes to decode the last character. This error can occur
          if bytes are read from a file in chunks or if binaries in other
          ways are split on non-UTF character boundaries. An <c>incomplete</c>
          tuple is then returned instead of the <c>error</c> tuple.
          It consists of the same parts as the <c>error</c> tuple, but
          the tag is <c>incomplete</c> instead of <c>error</c> and the
          last element is always guaranteed to be a binary consisting of
          the first part of a (so far) valid UTF character.</p>
        <p>If one UTF character is split over two consecutive binaries in
          the <c><anno>Data</anno></c>, the conversion succeeds. This means
          that a character can be decoded from a range of binaries as long
          as the whole range is specified as input without errors occurring.</p>
        <p><em>Example:</em></p>
        <code>
decode_data(Data) ->
   case unicode:characters_to_list(Data,unicode) of
      {incomplete,Encoded, Rest} ->
            More = get_some_more_data(),
            Encoded ++ decode_data([Rest, More]);
      {error,Encoded,Rest} ->
            handle_error(Encoded,Rest);
      List ->
            List
   end.</code>
        <p>However, bit strings that are not whole bytes are not allowed,
          so a UTF character must be split along 8-bit boundaries to
          ever be decoded.</p>
        <p>A <c>badarg</c> exception is thrown for the following cases:</p>
        <list type="bulleted">
          <item>Any parameters are of the wrong type.</item>
          <item>The list structure is invalid (a number as tail).</item>
          <item>The binaries do not contain whole bytes (bit strings).</item>
        </list>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfc_list" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a list of canonical equivalent
      composed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of canonical equivalent Composed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.
	</p>
	<p>The result is a list of characters.</p>
        <code>
3> unicode:characters_to_nfc_list([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776]]).
"abc..åäö"
</code>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfc_binary" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a utf8 binary of canonical equivalent
      composed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of canonical equivalent Composed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.</p>
	<p>The result is an utf8 encoded binary.</p>
        <code>
4> unicode:characters_to_nfc_binary([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776]]).
&lt;&lt;"abc..åäö"/utf8>>
</code>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfd_list" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a list of canonical equivalent
      decomposed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of canonical equivalent Decomposed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.
	</p>
	<p>The result is a list of characters.</p>
        <code>
1> unicode:characters_to_nfd_list("abc..åäö").
[97,98,99,46,46,97,778,97,776,111,776]
</code>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfd_binary" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a utf8 binary of canonical equivalent
      decomposed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of canonical equivalent Decomposed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.</p>
	<p>The result is an utf8 encoded binary.</p>
        <code>
2> unicode:characters_to_nfd_binary("abc..åäö").
&lt;&lt;97,98,99,46,46,97,204,138,97,204,136,111,204,136>>
</code>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfkc_list" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a list of canonical equivalent
      composed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of compatibly equivalent Composed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.
	</p>
	<p>The result is a list of characters.</p>
        <code>
3> unicode:characters_to_nfkc_list([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776],[65299,65298]]).
"abc..åäö32"
</code>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfkc_binary" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a utf8 binary of compatibly equivalent
      composed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of compatibly equivalent Composed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.</p>
	<p>The result is an utf8 encoded binary.</p>
        <code>
4> unicode:characters_to_nfkc_binary([&lt;&lt;"abc..a">>,[778],$a,[776],$o,[776],[65299,65298]]).
&lt;&lt;"abc..åäö32"/utf8>>
</code>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfkd_list" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a list of compatibly equivalent
      decomposed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of compatibly equivalent Decomposed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.
	</p>
	<p>The result is a list of characters.</p>
        <code>
1> unicode:characters_to_nfkd_list(["abc..åäö",[65299,65298]]).
[97,98,99,46,46,97,778,97,776,111,776,51,50]
</code>
      </desc>
    </func>

    <func>
      <name name="characters_to_nfkd_binary" arity="1" since="OTP 20.0"/>
      <fsummary>Normalize characters to a utf8 binary of compatibly equivalent
      decomposed Unicode characters.</fsummary>
      <desc>
        <p>Converts a possibly deep list of characters and binaries
        into a Normalized Form of compatibly equivalent Decomposed
        characters according to the Unicode standard.</p>
	<p>Any binaries in the input must be encoded with utf8
        encoding.</p>
	<p>The result is an utf8 encoded binary.</p>
        <code>
2> unicode:characters_to_nfkd_binary(["abc..åäö",[65299,65298]]).
&lt;&lt;97,98,99,46,46,97,204,138,97,204,136,111,204,136,51,50>>
</code>
      </desc>
    </func>

    <func>
      <name name="encoding_to_bom" arity="1"/>
      <fsummary>Create a binary UTF byte order mark from encoding.</fsummary>
      <type_desc variable="Bin">
        A <c>binary()</c> such that <c>byte_size(<anno>Bin</anno>) >= 4</c>.
      </type_desc>
      <desc>
        <p>Creates a UTF Byte Order Mark (BOM) as a binary from the
          supplied <c><anno>InEncoding</anno></c>. The BOM is, if supported at
          all, expected to be placed first in UTF encoded files or messages.</p>
        <p>The function returns <c>&lt;&lt;&gt;&gt;</c> for
          <c>latin1</c> encoding, as there is no BOM for ISO Latin-1.</p>
        <p>Notice that the BOM for UTF-8 is seldom used, and it
          is really not a <em>byte order</em> mark. There are obviously no
          byte order issues with UTF-8, so the BOM is only there to
          differentiate UTF-8 encoding from other UTF formats.</p>
      </desc>
    </func>
  </funcs>
</erlref>