From b507cbfdee5a553c6b24d8d69198786cfe11e361 Mon Sep 17 00:00:00 2001 From: Raimo Niskanen Date: Thu, 8 Dec 2011 15:56:11 +0100 Subject: Driver API: Update documentation for 64-bit sizes --- erts/doc/src/driver.xml | 10 +- erts/doc/src/driver_entry.xml | 48 ++++--- erts/doc/src/erl_driver.xml | 307 +++++++++++++++++++++++++++++++++++------- 3 files changed, 294 insertions(+), 71 deletions(-) diff --git a/erts/doc/src/driver.xml b/erts/doc/src/driver.xml index 9f246c4a6c..ac5729880d 100644 --- a/erts/doc/src/driver.xml +++ b/erts/doc/src/driver.xml @@ -30,11 +30,11 @@

This document was written a long time ago. A lot of it is still - valid, but some things have changed since it was first written. - Updates of this document are planned for the future. The reader - is encouraged to also read the - erl_driver, and the - driver_entry documentation. + interesting since it explains important concepts, but it was + written for an older driver interface so the examples does not + work anymore. The reader is encouraged to read + erl_driver and the + driver_entry documentation.

diff --git a/erts/doc/src/driver_entry.xml b/erts/doc/src/driver_entry.xml index 8bdd154cb9..a2efdf3ebc 100644 --- a/erts/doc/src/driver_entry.xml +++ b/erts/doc/src/driver_entry.xml @@ -34,18 +34,22 @@ driver_entry The driver-entry structure used by erlang drivers. -

As of erts version 5.5.3 the driver interface has been extended - (see extended marker). - The extended interface introduces - version management, - the possibility to pass capability flags - (see driver flags) - to the runtime system at driver initialization, and some new - driver API functions.

+

+ As of erts version 5.9 (OTP release R15B) the driver interface + has been changed with larger types for the callbacks + output, + control and + call. + See driver + version management in + erl_driver. +

Old drivers (compiled with an erl_driver.h from an - earlier erts version than 5.5.3) have to be recompiled - (but do not have to use the extended interface).

+ earlier erts version than 5.9) have to be updated and have + to use the extended interface (with + version management + ).

The driver_entry structure is a C struct that all erlang drivers define. It contains entry points for the erlang driver @@ -53,7 +57,7 @@ the driver.

- The erl_driver driver + The erl_driver driver API functions need a port handle that identifies the driver instance (and the port in the emulator). This is only passed to the start function, but @@ -84,7 +88,7 @@ the emulator, the driver is not allowed to modify the driver_entry.

-

Do not declare the driver_entryconst. This since the emulator needs to +

Do not declare the driver_entry const. This since the emulator needs to modify the handle, and the handle2 fields. A statically allocated, and const declared driver_entry may be located in @@ -116,7 +120,7 @@ typedef struct erl_drv_entry { void (*stop)(ErlDrvData drv_data); /* called when port is closed, and when the emulator is halted. */ - void (*output)(ErlDrvData drv_data, char *buf, int len); + void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len); /* called when we have output from erlang to the port */ void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); @@ -130,8 +134,9 @@ typedef struct erl_drv_entry { void (*finish)(void); /* called before unloading the driver - DYNAMIC DRIVERS ONLY */ void *handle; /* Reserved -- Used by emulator internally */ - int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, - int len, char **rbuf, int rlen); + ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command, + char *buf, ErlDrvSizeT len, + char **rbuf, ErlDrvSizeT rlen); /* "ioctl" for drivers - invoked by port_control/3 */ void (*timeout)(ErlDrvData drv_data); /* Handling of timeout in driver */ @@ -144,8 +149,9 @@ typedef struct erl_drv_entry { closed, and there is data in the driver queue that needs to be flushed before 'stop' can be called */ - int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, - int len, char **rbuf, int rlen, unsigned int *flags); + ErlDrvSSizeT (*call)(ErlDrvData drv_data, unsigned int command, + char *buf, ErlDrvSizeT len, + char **rbuf, ErlDrvSizeT rlen, unsigned int *flags); /* Works mostly like 'control', a synchronous call into the driver. */ void (*event)(ErlDrvData drv_data, ErlDrvEvent event, @@ -192,7 +198,7 @@ typedef struct erl_drv_entry { start, then stop is the place to deallocate that memory.

- void (*output)(ErlDrvData drv_data, char *buf, int len) + void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)

This is called when an erlang process has sent data to the port. The data is pointed to by buf, and is @@ -243,7 +249,7 @@ typedef struct erl_drv_entry { emulator will modify this field; therefore, it is important that the driver_entry isn't declared const.

 - int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, int len, char **rbuf, int rlen) + ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)

This is a special routine invoked with the erlang function port_control/3. It works a little like an "ioctl" for @@ -316,7 +322,7 @@ typedef struct erl_drv_entry { opposed to the asynchronous function, which is called in some thread (if multithreading is enabled).

 - int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, int len, char **rbuf, int rlen, unsigned int *flags) + ErlDrvSSizeT (*call)(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen, unsigned int *flags)

This function is called from erlang:port_call/3. It works a lot like the control call-back, but uses the @@ -452,7 +458,7 @@ typedef struct erl_drv_entry { SEE ALSO

erl_driver(3), erl_ddll(3), - erlang(3), + erlang(3), kernel(3)

diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 8e18dd6657..b5df4ca0c8 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -37,15 +37,18 @@

As of erts version 5.5.3 the driver interface has been extended (see extended marker). The extended interface introduce - version management, + version management, the possibility to pass capability flags (see driver flags) to the runtime system at driver initialization, and some new driver API functions.

-

Old drivers (compiled with an erl_driver.h from an - earlier erts version than 5.5.3) have to be recompiled - (but does not have to use the extended interface).

+

As of erts version 5.9 old drivers has to be recompiled + and has to use the extended interface. They also has to be + adjusted to the + 64-bit capable driver interface. + +

The driver calls back to the emulator, using the API functions declared in erl_driver.h. They are used for @@ -242,9 +245,9 @@

Adding / removing drivers - A driver can add and later remove drivers. +

A driver can add and later remove drivers.

 Monitoring processes - A driver can monitor a process that does not own a port. +

A driver can monitor a process that does not own a port.

 Version management

Version management is enabled for drivers that have set the @@ -268,15 +271,203 @@ versions differ, or if the major versions are equal and the minor version used by the driver is greater than the one used by the runtime system.

-

The emulator tries to check that a driver that doesn't use the - extended driver interface isn't incompatible when loading it. - It can, however, not make sure that it isn't incompatible. Therefore, - when loading a driver that doesn't use the extended driver - interface, there is a risk that it will be loaded also when - the driver is incompatible. When the driver uses the extended driver - interface, the emulator can verify that it isn't of an incompatible - driver version. You are therefore advised to use the extended driver - interface.

+

The emulator will refuse to load a driver that does not use + the extended driver interface since, + to allow for 64-bit capable drivers, + incompatible type changes for the callbacks + output, + control and + call + were introduced in release R15B. A driver written + with the old types would compile with warnings and when + called return garbage sizes to the emulator causing it + to read random memory and create huge incorrect result blobs.

+

Therefore it is not enough to just recompile drivers written with + version management for pre-R15B types; the types has to be changed + in the driver suggesting other rewrites especially regarding + size variables. Investigate all warnings when recompiling!

+

Also, the API driver functions driver_output*, + driver_vec_to_buf, driver_alloc/realloc* + and the driver_* queue functions were changed to have + larger length arguments and return values. This is a + lesser problem since code that passes smaller types + will get them auto converted in the calls and as long as + the driver does not handle sizes that overflow an int + all will work as before.

+ + + + +
+ + + REWRITES FOR 64-BIT DRIVER INTERFACE + +

+ For erts-5.9 two new integer types + ErlDrvSizeT and + ErlDrvSSizeT + were introduced that can hold 64-bit sizes if necessary. +

+

+ To not update a driver and just recompile it probably works + when building for a 32-bit machine creating a false sense of security. + Hopefully that will generate many important warnings. + But when recompiling the same driver later on for a 64-bit machine + there will be warnings and almost certainly crashes. + So it is a BAD idea to postpone updating the driver and + not fixing the warnings! +

+

+ When recompiling with gcc use the -Wstrict-prototypes + flag to get better warnings. Try to find a similar flag if you + are using some other compiler. +

+

+ Here follows a checklist for rewriting a pre erts-5.9 driver, + most important first. +

+ + Return types for driver callbacks + +

+ Rewrite driver callback + control + to use return type ErlDrvSSizeT instead of int. +

+

+ Rewrite driver callback + call + to use return type ErlDrvSSizeT instead of int. +

+ +

+ These changes are essential to not crash the emulator + or worse cause malfunction. + Without them a driver may return garbage in the high 32 bits + to the emulator causing it to build a huge result from random + bytes either crashing on memory allocation or succeeding with + a random result from the driver call. +

+ + + Arguments to driver callbacks + +

+ Driver callback + output + now gets ErlDrvSizeT as 3:rd argument instead + of previously int. +

+

+ Driver callback + control + now gets ErlDrvSizeT as 4:th and 6:th arguments instead + of previously int. +

+

+ Driver callback + call + now gets ErlDrvSizeT as 4:th and 6:th arguments instead + of previously int. +

+

+ Sane compiler's calling conventions probably make these changes + necessary only for a driver to handle data chunks that require + 64-bit size fields (mostly larger than 2 GB since that is what + an int of 32 bits can hold). But it is possible to think + of non-sane calling conventions that would make the driver + callbacks mix up the arguments causing malfunction. +

+ +

+ The argument type change is from signed to unsigned which + may cause problems for e.g. loop termination conditions or + error conditions if you just change the types all over the place. +

+ + + Larger size field in ErlIOVec + +

+ The size field in + ErlIOVec + has been changed to ErlDrvSizeT from int. + Check all code that use that field. +

+

+ Automatic type casting probably makes these changes necessary only + for a driver that encounters sizes larger than 32 bits. +

+ +

+ The size field changed from signed to unsigned which + may cause problems for e.g. loop termination conditions or + error conditions if you just change the types all over the place. +

+ + + Arguments and return values in the driver API + +

+ Many driver API functions has changed argument type + and/or return value to ErlDrvSizeT from mostly int. + Automatic type casting probably makes these changes necessary only + for a driver that encounters sizes larger than 32 bits. +

+ + driver_output + 3:rd argument + driver_output2 + 3:rd and 5:th arguments + + driver_output_binary + + 3:rd 5:th and 6:th arguments + driver_outputv + 3:rd and 5:th arguments + + driver_vec_to_buf + + 3:rd argument and return value + driver_alloc + 1:st argument + driver_realloc + 2:nd argument + + driver_alloc_binary + + 1:st argument + + driver_realloc_binary + + 2:nd argument + driver_enq + 3:rd argument + driver_pushq + 3:rd argument + driver_deq + 2:nd argument and return value + driver_sizeq + return value + driver_enq_bin + 3:rd and 4:th argument + driver_pushq_bin + 3:rd and 4:th argument + driver_enqv + 3:rd argument + driver_pushqv + 3:rd argument + driver_peekqv + return value + + +

+ This is a change from signed to unsigned which + may cause problems for e.g. loop termination conditions and + error conditions if you just change the types all over the place. +

+
@@ -285,7 +476,11 @@ DATA TYPES - ErlDrvSysInfo + ErlDrvSizeT +

An unsigned integer type to be used as size_t

 + ErlDrvSSizeT +

A signed integer type the size of ErlDrvSizeT

 + ErlDrvSysInfo

@@ -332,12 +527,12 @@ typedef struct ErlDrvSysInfo { erts_version A string containing the version number of the runtime system (the same as returned by - erlang:system_info(version)). + erlang:system_info(version)). otp_release A string containing the OTP release number (the same as returned by - erlang:system_info(otp_release)). + erlang:system_info(otp_release)). thread_support A value != 0 if the runtime system has thread support; @@ -351,12 +546,12 @@ typedef struct ErlDrvSysInfo { The number of async threads in the async thread pool used by driver_async() (the same as returned by - erlang:system_info(thread_pool_size)). + erlang:system_info(thread_pool_size)). scheduler_threads The number of scheduler threads used by the runtime system (the same as returned by - erlang:system_info(schedulers)). + erlang:system_info(schedulers)). nif_major_version The value of ERL_NIF_MAJOR_VERSION when the runtime system was compiled. @@ -371,7 +566,7 @@ typedef struct ErlDrvSysInfo {

typedef struct ErlDrvBinary { - int orig_size; + ErlDrvSint orig_size; char orig_bytes[]; } ErlDrvBinary; @@ -423,7 +618,7 @@ typedef struct ErlDrvBinary {

The ErlDrvData is a handle to driver-specific data, passed to the driver call-backs. It is a pointer, and is - most often casted to a specific pointer in the driver.

+ most often type casted to a specific pointer in the driver.

 SysIOVec @@ -437,7 +632,7 @@ typedef struct ErlDrvBinary { typedef struct ErlIOVec { int vsize; - int size; + ErlDrvSizeT size; SysIOVec* iov; ErlDrvBinary** binv; } ErlIOVec; @@ -630,7 +825,7 @@ typedef struct ErlIOVec { - intdriver_output(ErlDrvPort port, char *buf, int len) + intdriver_output(ErlDrvPort port, char *buf, ErlDrvSizeT len) Send data from driver to port owner @@ -650,7 +845,7 @@ typedef struct ErlIOVec { - intdriver_output2(ErlDrvPort port, char *hbuf, int hlen, char *buf, int len) + intdriver_output2(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen, char *buf, ErlDrvSizeT len) Send data and binary data to port owner @@ -665,7 +860,7 @@ typedef struct ErlIOVec { - intdriver_output_binary(ErlDrvPort port, char *hbuf, int hlen, ErlDrvBinary* bin, int offset, int len) + intdriver_output_binary(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen, ErlDrvBinary* bin, ErlDrvSizeT offset, ErlDrvSizeT len) Send data from a driver binary to port owner @@ -688,7 +883,7 @@ typedef struct ErlIOVec { - intdriver_outputv(ErlDrvPort port, char* hbuf, int hlen, ErlIOVec *ev, int skip) + intdriver_outputv(ErlDrvPort port, char* hbuf, ErlDrvSizeT hlen, ErlIOVec *ev, ErlDrvSizeT skip) Send vectorized data to port owner @@ -711,7 +906,7 @@ typedef struct ErlIOVec { - intdriver_vec_to_buf(ErlIOVec *ev, char *buf, int len) + ErlDrvSizeTdriver_vec_to_buf(ErlIOVec *ev, char *buf, ErlDrvSizeT len) Collect data segments into a buffer @@ -738,7 +933,7 @@ typedef struct ErlIOVec { time parameter is the time in milliseconds before the timer expires.

When the timer reaches 0 and expires, the driver entry - function timeout is called.

+ function timeout is called.

Note that there is only one timer on each driver instance; setting a new timer will replace an older one.

Return value is 0 (-1 only when the timeout driver @@ -832,7 +1027,7 @@ typedef struct ErlIOVec { - void *driver_alloc(size_t size) + void *driver_alloc(ErlDrvSizeT size) Allocate memory @@ -846,7 +1041,7 @@ typedef struct ErlIOVec { - void *driver_realloc(void *ptr, size_t size) + void *driver_realloc(void *ptr, ErlDrvSizeT size) Resize an allocated memory block @@ -872,7 +1067,7 @@ typedef struct ErlIOVec { - ErlDrvBinary*driver_alloc_binary(int size) + ErlDrvBinary*driver_alloc_binary(ErlDrvSizeT size) Allocate a driver binary @@ -892,7 +1087,7 @@ typedef struct ErlIOVec { - ErlDrvBinary*driver_realloc_binary(ErlDrvBinary *bin, int size) + ErlDrvBinary*driver_realloc_binary(ErlDrvBinary *bin, ErlDrvSizeT size) Resize a driver binary @@ -958,7 +1153,7 @@ typedef struct ErlIOVec { - intdriver_enq(ErlDrvPort port, char* buf, int len) + intdriver_enq(ErlDrvPort port, char* buf, ErlDrvSizeT len) Enqueue data in the driver queue @@ -982,7 +1177,7 @@ typedef struct ErlIOVec { - intdriver_pushq(ErlDrvPort port, char* buf, int len) + intdriver_pushq(ErlDrvPort port, char* buf, ErlDrvSizeT len) Push data at the head of the driver queue @@ -997,7 +1192,7 @@ typedef struct ErlIOVec { - intdriver_deq(ErlDrvPort port, int size) + ErlDrvSizeTdriver_deq(ErlDrvPort port, ErlDrvSizeT size) Dequeue data from the head of the driver queue @@ -1013,7 +1208,7 @@ typedef struct ErlIOVec { - intdriver_sizeq(ErlDrvPort port) + ErlDrvSizeTdriver_sizeq(ErlDrvPort port) Return the size of the driver queue @@ -1026,7 +1221,7 @@ typedef struct ErlIOVec { - intdriver_enq_bin(ErlDrvPort port, ErlDrvBinary *bin, int offset, int len) + intdriver_enq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset, ErlDrvSizeT len) Enqueue binary in the driver queue @@ -1043,7 +1238,7 @@ typedef struct ErlIOVec { - intdriver_pushq_bin(ErlDrvPort port, ErlDrvBinary *bin, int offset, int len) + intdriver_pushq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset, ErlDrvSizeT len) Push binary at the head of the driver queue @@ -1059,6 +1254,28 @@ typedef struct ErlIOVec {

The return value is 0.

 + + ErlDrvSizeTdriver_peekqv(ErlDrvPort port, ErlIOVec *ev) + Get the driver queue as an IO vector + + +

+ This function retrieves the driver queue into a supplied + ErlIOVec ev. It also returns the queue size. + This is one of two ways to get data out of the queue. +

+

+ If ev is NULL all ones i.e. -1 type cast to + ErlDrvSizeT is returned. +

+

Nothing is removed from the queue by this function, that must be done + with driver_deq.

+

This function can be called from an arbitrary thread if a + port data lock + associated with the port is locked by the calling + thread during the call.

+ + SysIOVec*driver_peekq(ErlDrvPort port, int *vlen) Get the driver queue as a vector @@ -1066,7 +1283,7 @@ typedef struct ErlIOVec {

This function retrieves the driver queue as a pointer to an array of SysIOVecs. It also returns the number of - elements in vlen. This is the only way to get data + elements in vlen. This is one of two ways to get data out of the queue.

Nothing is removed from the queue by this function, that must be done with driver_deq.

@@ -1079,7 +1296,7 @@ typedef struct ErlIOVec { - intdriver_enqv(ErlDrvPort port, ErlIOVec *ev, int skip) + intdriver_enqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip) Enqueue vector in the driver queue @@ -1095,7 +1312,7 @@ typedef struct ErlIOVec { - intdriver_pushqv(ErlDrvPort port, ErlIOVec *ev, int skip) + intdriver_pushqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip) Push vector at the head of the driver queue @@ -1494,7 +1711,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len term encoded with the external format, i.e., a term that has been encoded by - erlang:term_to_binary, + erlang:term_to_binary, erl_interface, etc. For example, if binp is a pointer to an ErlDrvBinary that contains the term {17, 4711} encoded with the @@ -1694,7 +1911,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len The driver defined handle is normally created in the driver start call-back when a port is created via - erlang:open_port/2. + erlang:open_port/2.

The caller of driver_create_port() is allowed to manipulate the newly created port when driver_create_port() @@ -2462,7 +2679,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len SEE ALSO

driver_entry(3), erl_ddll(3), - erlang(3)

+ erlang(3)

An Alternative Distribution Driver (ERTS User's Guide Ch. 3)

-- cgit v1.2.3