diff options
Diffstat (limited to 'lib/inets')
49 files changed, 4770 insertions, 1846 deletions
diff --git a/lib/inets/Makefile b/lib/inets/Makefile index f4c2746b0a..4765a2ca3c 100644 --- a/lib/inets/Makefile +++ b/lib/inets/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2010. All Rights Reserved. +# Copyright Ericsson AB 1996-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/inets/doc/archive/rfc3986.txt b/lib/inets/doc/archive/rfc3986.txt new file mode 100644 index 0000000000..c56ed4eb70 --- /dev/null +++ b/lib/inets/doc/archive/rfc3986.txt @@ -0,0 +1,3419 @@ + + + + + + +Network Working Group T. Berners-Lee +Request for Comments: 3986 W3C/MIT +STD: 66 R. Fielding +Updates: 1738 Day Software +Obsoletes: 2732, 2396, 1808 L. Masinter +Category: Standards Track Adobe Systems + January 2005 + + + Uniform Resource Identifier (URI): Generic Syntax + +Status of This Memo + + This document specifies an Internet standards track protocol for the + Internet community, and requests discussion and suggestions for + improvements. Please refer to the current edition of the "Internet + Official Protocol Standards" (STD 1) for the standardization state + and status of this protocol. Distribution of this memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2005). + +Abstract + + A Uniform Resource Identifier (URI) is a compact sequence of + characters that identifies an abstract or physical resource. This + specification defines the generic URI syntax and a process for + resolving URI references that might be in relative form, along with + guidelines and security considerations for the use of URIs on the + Internet. The URI syntax defines a grammar that is a superset of all + valid URIs, allowing an implementation to parse the common components + of a URI reference without knowing the scheme-specific requirements + of every possible identifier. This specification does not define a + generative grammar for URIs; that task is performed by the individual + specifications of each URI scheme. + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 1] + +RFC 3986 URI Generic Syntax January 2005 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 1.1. Overview of URIs . . . . . . . . . . . . . . . . . . . . 4 + 1.1.1. Generic Syntax . . . . . . . . . . . . . . . . . 6 + 1.1.2. Examples . . . . . . . . . . . . . . . . . . . . 7 + 1.1.3. URI, URL, and URN . . . . . . . . . . . . . . . 7 + 1.2. Design Considerations . . . . . . . . . . . . . . . . . 8 + 1.2.1. Transcription . . . . . . . . . . . . . . . . . 8 + 1.2.2. Separating Identification from Interaction . . . 9 + 1.2.3. Hierarchical Identifiers . . . . . . . . . . . . 10 + 1.3. Syntax Notation . . . . . . . . . . . . . . . . . . . . 11 + 2. Characters . . . . . . . . . . . . . . . . . . . . . . . . . . 11 + 2.1. Percent-Encoding . . . . . . . . . . . . . . . . . . . . 12 + 2.2. Reserved Characters . . . . . . . . . . . . . . . . . . 12 + 2.3. Unreserved Characters . . . . . . . . . . . . . . . . . 13 + 2.4. When to Encode or Decode . . . . . . . . . . . . . . . . 14 + 2.5. Identifying Data . . . . . . . . . . . . . . . . . . . . 14 + 3. Syntax Components . . . . . . . . . . . . . . . . . . . . . . 16 + 3.1. Scheme . . . . . . . . . . . . . . . . . . . . . . . . . 17 + 3.2. Authority . . . . . . . . . . . . . . . . . . . . . . . 17 + 3.2.1. User Information . . . . . . . . . . . . . . . . 18 + 3.2.2. Host . . . . . . . . . . . . . . . . . . . . . . 18 + 3.2.3. Port . . . . . . . . . . . . . . . . . . . . . . 22 + 3.3. Path . . . . . . . . . . . . . . . . . . . . . . . . . . 22 + 3.4. Query . . . . . . . . . . . . . . . . . . . . . . . . . 23 + 3.5. Fragment . . . . . . . . . . . . . . . . . . . . . . . . 24 + 4. Usage . . . . . . . . . . . . . . . . . . . . . . . . . . . . 25 + 4.1. URI Reference . . . . . . . . . . . . . . . . . . . . . 25 + 4.2. Relative Reference . . . . . . . . . . . . . . . . . . . 26 + 4.3. Absolute URI . . . . . . . . . . . . . . . . . . . . . . 27 + 4.4. Same-Document Reference . . . . . . . . . . . . . . . . 27 + 4.5. Suffix Reference . . . . . . . . . . . . . . . . . . . . 27 + 5. Reference Resolution . . . . . . . . . . . . . . . . . . . . . 28 + 5.1. Establishing a Base URI . . . . . . . . . . . . . . . . 28 + 5.1.1. Base URI Embedded in Content . . . . . . . . . . 29 + 5.1.2. Base URI from the Encapsulating Entity . . . . . 29 + 5.1.3. Base URI from the Retrieval URI . . . . . . . . 30 + 5.1.4. Default Base URI . . . . . . . . . . . . . . . . 30 + 5.2. Relative Resolution . . . . . . . . . . . . . . . . . . 30 + 5.2.1. Pre-parse the Base URI . . . . . . . . . . . . . 31 + 5.2.2. Transform References . . . . . . . . . . . . . . 31 + 5.2.3. Merge Paths . . . . . . . . . . . . . . . . . . 32 + 5.2.4. Remove Dot Segments . . . . . . . . . . . . . . 33 + 5.3. Component Recomposition . . . . . . . . . . . . . . . . 35 + 5.4. Reference Resolution Examples . . . . . . . . . . . . . 35 + 5.4.1. Normal Examples . . . . . . . . . . . . . . . . 36 + 5.4.2. Abnormal Examples . . . . . . . . . . . . . . . 36 + + + +Berners-Lee, et al. Standards Track [Page 2] + +RFC 3986 URI Generic Syntax January 2005 + + + 6. Normalization and Comparison . . . . . . . . . . . . . . . . . 38 + 6.1. Equivalence . . . . . . . . . . . . . . . . . . . . . . 38 + 6.2. Comparison Ladder . . . . . . . . . . . . . . . . . . . 39 + 6.2.1. Simple String Comparison . . . . . . . . . . . . 39 + 6.2.2. Syntax-Based Normalization . . . . . . . . . . . 40 + 6.2.3. Scheme-Based Normalization . . . . . . . . . . . 41 + 6.2.4. Protocol-Based Normalization . . . . . . . . . . 42 + 7. Security Considerations . . . . . . . . . . . . . . . . . . . 43 + 7.1. Reliability and Consistency . . . . . . . . . . . . . . 43 + 7.2. Malicious Construction . . . . . . . . . . . . . . . . . 43 + 7.3. Back-End Transcoding . . . . . . . . . . . . . . . . . . 44 + 7.4. Rare IP Address Formats . . . . . . . . . . . . . . . . 45 + 7.5. Sensitive Information . . . . . . . . . . . . . . . . . 45 + 7.6. Semantic Attacks . . . . . . . . . . . . . . . . . . . . 45 + 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 46 + 9. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . . 46 + 10. References . . . . . . . . . . . . . . . . . . . . . . . . . . 46 + 10.1. Normative References . . . . . . . . . . . . . . . . . . 46 + 10.2. Informative References . . . . . . . . . . . . . . . . . 47 + A. Collected ABNF for URI . . . . . . . . . . . . . . . . . . . . 49 + B. Parsing a URI Reference with a Regular Expression . . . . . . 50 + C. Delimiting a URI in Context . . . . . . . . . . . . . . . . . 51 + D. Changes from RFC 2396 . . . . . . . . . . . . . . . . . . . . 53 + D.1. Additions . . . . . . . . . . . . . . . . . . . . . . . 53 + D.2. Modifications . . . . . . . . . . . . . . . . . . . . . 53 + Index . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 56 + Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . . 60 + Full Copyright Statement . . . . . . . . . . . . . . . . . . . . . 61 + + + + + + + + + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 3] + +RFC 3986 URI Generic Syntax January 2005 + + +1. Introduction + + A Uniform Resource Identifier (URI) provides a simple and extensible + means for identifying a resource. This specification of URI syntax + and semantics is derived from concepts introduced by the World Wide + Web global information initiative, whose use of these identifiers + dates from 1990 and is described in "Universal Resource Identifiers + in WWW" [RFC1630]. The syntax is designed to meet the + recommendations laid out in "Functional Recommendations for Internet + Resource Locators" [RFC1736] and "Functional Requirements for Uniform + Resource Names" [RFC1737]. + + This document obsoletes [RFC2396], which merged "Uniform Resource + Locators" [RFC1738] and "Relative Uniform Resource Locators" + [RFC1808] in order to define a single, generic syntax for all URIs. + It obsoletes [RFC2732], which introduced syntax for an IPv6 address. + It excludes portions of RFC 1738 that defined the specific syntax of + individual URI schemes; those portions will be updated as separate + documents. The process for registration of new URI schemes is + defined separately by [BCP35]. Advice for designers of new URI + schemes can be found in [RFC2718]. All significant changes from RFC + 2396 are noted in Appendix D. + + This specification uses the terms "character" and "coded character + set" in accordance with the definitions provided in [BCP19], and + "character encoding" in place of what [BCP19] refers to as a + "charset". + +1.1. Overview of URIs + + URIs are characterized as follows: + + Uniform + + Uniformity provides several benefits. It allows different types + of resource identifiers to be used in the same context, even when + the mechanisms used to access those resources may differ. It + allows uniform semantic interpretation of common syntactic + conventions across different types of resource identifiers. It + allows introduction of new types of resource identifiers without + interfering with the way that existing identifiers are used. It + allows the identifiers to be reused in many different contexts, + thus permitting new applications or protocols to leverage a pre- + existing, large, and widely used set of resource identifiers. + + + + + + + +Berners-Lee, et al. Standards Track [Page 4] + +RFC 3986 URI Generic Syntax January 2005 + + + Resource + + This specification does not limit the scope of what might be a + resource; rather, the term "resource" is used in a general sense + for whatever might be identified by a URI. Familiar examples + include an electronic document, an image, a source of information + with a consistent purpose (e.g., "today's weather report for Los + Angeles"), a service (e.g., an HTTP-to-SMS gateway), and a + collection of other resources. A resource is not necessarily + accessible via the Internet; e.g., human beings, corporations, and + bound books in a library can also be resources. Likewise, + abstract concepts can be resources, such as the operators and + operands of a mathematical equation, the types of a relationship + (e.g., "parent" or "employee"), or numeric values (e.g., zero, + one, and infinity). + + Identifier + + An identifier embodies the information required to distinguish + what is being identified from all other things within its scope of + identification. Our use of the terms "identify" and "identifying" + refer to this purpose of distinguishing one resource from all + other resources, regardless of how that purpose is accomplished + (e.g., by name, address, or context). These terms should not be + mistaken as an assumption that an identifier defines or embodies + the identity of what is referenced, though that may be the case + for some identifiers. Nor should it be assumed that a system + using URIs will access the resource identified: in many cases, + URIs are used to denote resources without any intention that they + be accessed. Likewise, the "one" resource identified might not be + singular in nature (e.g., a resource might be a named set or a + mapping that varies over time). + + A URI is an identifier consisting of a sequence of characters + matching the syntax rule named <URI> in Section 3. It enables + uniform identification of resources via a separately defined + extensible set of naming schemes (Section 3.1). How that + identification is accomplished, assigned, or enabled is delegated to + each scheme specification. + + This specification does not place any limits on the nature of a + resource, the reasons why an application might seek to refer to a + resource, or the kinds of systems that might use URIs for the sake of + identifying resources. This specification does not require that a + URI persists in identifying the same resource over time, though that + is a common goal of all URI schemes. Nevertheless, nothing in this + + + + + +Berners-Lee, et al. Standards Track [Page 5] + +RFC 3986 URI Generic Syntax January 2005 + + + specification prevents an application from limiting itself to + particular types of resources, or to a subset of URIs that maintains + characteristics desired by that application. + + URIs have a global scope and are interpreted consistently regardless + of context, though the result of that interpretation may be in + relation to the end-user's context. For example, "http://localhost/" + has the same interpretation for every user of that reference, even + though the network interface corresponding to "localhost" may be + different for each end-user: interpretation is independent of access. + However, an action made on the basis of that reference will take + place in relation to the end-user's context, which implies that an + action intended to refer to a globally unique thing must use a URI + that distinguishes that resource from all other things. URIs that + identify in relation to the end-user's local context should only be + used when the context itself is a defining aspect of the resource, + such as when an on-line help manual refers to a file on the end- + user's file system (e.g., "file:///etc/hosts"). + +1.1.1. Generic Syntax + + Each URI begins with a scheme name, as defined in Section 3.1, that + refers to a specification for assigning identifiers within that + scheme. As such, the URI syntax is a federated and extensible naming + system wherein each scheme's specification may further restrict the + syntax and semantics of identifiers using that scheme. + + This specification defines those elements of the URI syntax that are + required of all URI schemes or are common to many URI schemes. It + thus defines the syntax and semantics needed to implement a scheme- + independent parsing mechanism for URI references, by which the + scheme-dependent handling of a URI can be postponed until the + scheme-dependent semantics are needed. Likewise, protocols and data + formats that make use of URI references can refer to this + specification as a definition for the range of syntax allowed for all + URIs, including those schemes that have yet to be defined. This + decouples the evolution of identification schemes from the evolution + of protocols, data formats, and implementations that make use of + URIs. + + A parser of the generic URI syntax can parse any URI reference into + its major components. Once the scheme is determined, further + scheme-specific parsing can be performed on the components. In other + words, the URI generic syntax is a superset of the syntax of all URI + schemes. + + + + + + +Berners-Lee, et al. Standards Track [Page 6] + +RFC 3986 URI Generic Syntax January 2005 + + +1.1.2. Examples + + The following example URIs illustrate several URI schemes and + variations in their common syntax components: + + ftp://ftp.is.co.za/rfc/rfc1808.txt + + http://www.ietf.org/rfc/rfc2396.txt + + ldap://[2001:db8::7]/c=GB?objectClass?one + + mailto:[email protected] + + news:comp.infosystems.www.servers.unix + + tel:+1-816-555-1212 + + telnet://192.0.2.16:80/ + + urn:oasis:names:specification:docbook:dtd:xml:4.1.2 + + +1.1.3. URI, URL, and URN + + A URI can be further classified as a locator, a name, or both. The + term "Uniform Resource Locator" (URL) refers to the subset of URIs + that, in addition to identifying a resource, provide a means of + locating the resource by describing its primary access mechanism + (e.g., its network "location"). The term "Uniform Resource Name" + (URN) has been used historically to refer to both URIs under the + "urn" scheme [RFC2141], which are required to remain globally unique + and persistent even when the resource ceases to exist or becomes + unavailable, and to any other URI with the properties of a name. + + An individual scheme does not have to be classified as being just one + of "name" or "locator". Instances of URIs from any given scheme may + have the characteristics of names or locators or both, often + depending on the persistence and care in the assignment of + identifiers by the naming authority, rather than on any quality of + the scheme. Future specifications and related documentation should + use the general term "URI" rather than the more restrictive terms + "URL" and "URN" [RFC3305]. + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 7] + +RFC 3986 URI Generic Syntax January 2005 + + +1.2. Design Considerations + +1.2.1. Transcription + + The URI syntax has been designed with global transcription as one of + its main considerations. A URI is a sequence of characters from a + very limited set: the letters of the basic Latin alphabet, digits, + and a few special characters. A URI may be represented in a variety + of ways; e.g., ink on paper, pixels on a screen, or a sequence of + character encoding octets. The interpretation of a URI depends only + on the characters used and not on how those characters are + represented in a network protocol. + + The goal of transcription can be described by a simple scenario. + Imagine two colleagues, Sam and Kim, sitting in a pub at an + international conference and exchanging research ideas. Sam asks Kim + for a location to get more information, so Kim writes the URI for the + research site on a napkin. Upon returning home, Sam takes out the + napkin and types the URI into a computer, which then retrieves the + information to which Kim referred. + + There are several design considerations revealed by the scenario: + + o A URI is a sequence of characters that is not always represented + as a sequence of octets. + + o A URI might be transcribed from a non-network source and thus + should consist of characters that are most likely able to be + entered into a computer, within the constraints imposed by + keyboards (and related input devices) across languages and + locales. + + o A URI often has to be remembered by people, and it is easier for + people to remember a URI when it consists of meaningful or + familiar components. + + These design considerations are not always in alignment. For + example, it is often the case that the most meaningful name for a URI + component would require characters that cannot be typed into some + systems. The ability to transcribe a resource identifier from one + medium to another has been considered more important than having a + URI consist of the most meaningful of components. + + In local or regional contexts and with improving technology, users + might benefit from being able to use a wider range of characters; + such use is not defined by this specification. Percent-encoded + octets (Section 2.1) may be used within a URI to represent characters + outside the range of the US-ASCII coded character set if this + + + +Berners-Lee, et al. Standards Track [Page 8] + +RFC 3986 URI Generic Syntax January 2005 + + + representation is allowed by the scheme or by the protocol element in + which the URI is referenced. Such a definition should specify the + character encoding used to map those characters to octets prior to + being percent-encoded for the URI. + +1.2.2. Separating Identification from Interaction + + A common misunderstanding of URIs is that they are only used to refer + to accessible resources. The URI itself only provides + identification; access to the resource is neither guaranteed nor + implied by the presence of a URI. Instead, any operation associated + with a URI reference is defined by the protocol element, data format + attribute, or natural language text in which it appears. + + Given a URI, a system may attempt to perform a variety of operations + on the resource, as might be characterized by words such as "access", + "update", "replace", or "find attributes". Such operations are + defined by the protocols that make use of URIs, not by this + specification. However, we do use a few general terms for describing + common operations on URIs. URI "resolution" is the process of + determining an access mechanism and the appropriate parameters + necessary to dereference a URI; this resolution may require several + iterations. To use that access mechanism to perform an action on the + URI's resource is to "dereference" the URI. + + When URIs are used within information retrieval systems to identify + sources of information, the most common form of URI dereference is + "retrieval": making use of a URI in order to retrieve a + representation of its associated resource. A "representation" is a + sequence of octets, along with representation metadata describing + those octets, that constitutes a record of the state of the resource + at the time when the representation is generated. Retrieval is + achieved by a process that might include using the URI as a cache key + to check for a locally cached representation, resolution of the URI + to determine an appropriate access mechanism (if any), and + dereference of the URI for the sake of applying a retrieval + operation. Depending on the protocols used to perform the retrieval, + additional information might be supplied about the resource (resource + metadata) and its relation to other resources. + + URI references in information retrieval systems are designed to be + late-binding: the result of an access is generally determined when it + is accessed and may vary over time or due to other aspects of the + interaction. These references are created in order to be used in the + future: what is being identified is not some specific result that was + obtained in the past, but rather some characteristic that is expected + to be true for future results. In such cases, the resource referred + to by the URI is actually a sameness of characteristics as observed + + + +Berners-Lee, et al. Standards Track [Page 9] + +RFC 3986 URI Generic Syntax January 2005 + + + over time, perhaps elucidated by additional comments or assertions + made by the resource provider. + + Although many URI schemes are named after protocols, this does not + imply that use of these URIs will result in access to the resource + via the named protocol. URIs are often used simply for the sake of + identification. Even when a URI is used to retrieve a representation + of a resource, that access might be through gateways, proxies, + caches, and name resolution services that are independent of the + protocol associated with the scheme name. The resolution of some + URIs may require the use of more than one protocol (e.g., both DNS + and HTTP are typically used to access an "http" URI's origin server + when a representation isn't found in a local cache). + +1.2.3. Hierarchical Identifiers + + The URI syntax is organized hierarchically, with components listed in + order of decreasing significance from left to right. For some URI + schemes, the visible hierarchy is limited to the scheme itself: + everything after the scheme component delimiter (":") is considered + opaque to URI processing. Other URI schemes make the hierarchy + explicit and visible to generic parsing algorithms. + + The generic syntax uses the slash ("/"), question mark ("?"), and + number sign ("#") characters to delimit components that are + significant to the generic parser's hierarchical interpretation of an + identifier. In addition to aiding the readability of such + identifiers through the consistent use of familiar syntax, this + uniform representation of hierarchy across naming schemes allows + scheme-independent references to be made relative to that hierarchy. + + It is often the case that a group or "tree" of documents has been + constructed to serve a common purpose, wherein the vast majority of + URI references in these documents point to resources within the tree + rather than outside it. Similarly, documents located at a particular + site are much more likely to refer to other resources at that site + than to resources at remote sites. Relative referencing of URIs + allows document trees to be partially independent of their location + and access scheme. For instance, it is possible for a single set of + hypertext documents to be simultaneously accessible and traversable + via each of the "file", "http", and "ftp" schemes if the documents + refer to each other with relative references. Furthermore, such + document trees can be moved, as a whole, without changing any of the + relative references. + + A relative reference (Section 4.2) refers to a resource by describing + the difference within a hierarchical name space between the reference + context and the target URI. The reference resolution algorithm, + + + +Berners-Lee, et al. Standards Track [Page 10] + +RFC 3986 URI Generic Syntax January 2005 + + + presented in Section 5, defines how such a reference is transformed + to the target URI. As relative references can only be used within + the context of a hierarchical URI, designers of new URI schemes + should use a syntax consistent with the generic syntax's hierarchical + components unless there are compelling reasons to forbid relative + referencing within that scheme. + + NOTE: Previous specifications used the terms "partial URI" and + "relative URI" to denote a relative reference to a URI. As some + readers misunderstood those terms to mean that relative URIs are a + subset of URIs rather than a method of referencing URIs, this + specification simply refers to them as relative references. + + All URI references are parsed by generic syntax parsers when used. + However, because hierarchical processing has no effect on an absolute + URI used in a reference unless it contains one or more dot-segments + (complete path segments of "." or "..", as described in Section 3.3), + URI scheme specifications can define opaque identifiers by + disallowing use of slash characters, question mark characters, and + the URIs "scheme:." and "scheme:..". + +1.3. Syntax Notation + + This specification uses the Augmented Backus-Naur Form (ABNF) + notation of [RFC2234], including the following core ABNF syntax rules + defined by that specification: ALPHA (letters), CR (carriage return), + DIGIT (decimal digits), DQUOTE (double quote), HEXDIG (hexadecimal + digits), LF (line feed), and SP (space). The complete URI syntax is + collected in Appendix A. + +2. Characters + + The URI syntax provides a method of encoding data, presumably for the + sake of identifying a resource, as a sequence of characters. The URI + characters are, in turn, frequently encoded as octets for transport + or presentation. This specification does not mandate any particular + character encoding for mapping between URI characters and the octets + used to store or transmit those characters. When a URI appears in a + protocol element, the character encoding is defined by that protocol; + without such a definition, a URI is assumed to be in the same + character encoding as the surrounding text. + + The ABNF notation defines its terminal values to be non-negative + integers (codepoints) based on the US-ASCII coded character set + [ASCII]. Because a URI is a sequence of characters, we must invert + that relation in order to understand the URI syntax. Therefore, the + + + + + +Berners-Lee, et al. Standards Track [Page 11] + +RFC 3986 URI Generic Syntax January 2005 + + + integer values used by the ABNF must be mapped back to their + corresponding characters via US-ASCII in order to complete the syntax + rules. + + A URI is composed from a limited set of characters consisting of + digits, letters, and a few graphic symbols. A reserved subset of + those characters may be used to delimit syntax components within a + URI while the remaining characters, including both the unreserved set + and those reserved characters not acting as delimiters, define each + component's identifying data. + +2.1. Percent-Encoding + + A percent-encoding mechanism is used to represent a data octet in a + component when that octet's corresponding character is outside the + allowed set or is being used as a delimiter of, or within, the + component. A percent-encoded octet is encoded as a character + triplet, consisting of the percent character "%" followed by the two + hexadecimal digits representing that octet's numeric value. For + example, "%20" is the percent-encoding for the binary octet + "00100000" (ABNF: %x20), which in US-ASCII corresponds to the space + character (SP). Section 2.4 describes when percent-encoding and + decoding is applied. + + pct-encoded = "%" HEXDIG HEXDIG + + The uppercase hexadecimal digits 'A' through 'F' are equivalent to + the lowercase digits 'a' through 'f', respectively. If two URIs + differ only in the case of hexadecimal digits used in percent-encoded + octets, they are equivalent. For consistency, URI producers and + normalizers should use uppercase hexadecimal digits for all percent- + encodings. + +2.2. Reserved Characters + + URIs include components and subcomponents that are delimited by + characters in the "reserved" set. These characters are called + "reserved" because they may (or may not) be defined as delimiters by + the generic syntax, by each scheme-specific syntax, or by the + implementation-specific syntax of a URI's dereferencing algorithm. + If data for a URI component would conflict with a reserved + character's purpose as a delimiter, then the conflicting data must be + percent-encoded before the URI is formed. + + + + + + + + +Berners-Lee, et al. Standards Track [Page 12] + +RFC 3986 URI Generic Syntax January 2005 + + + reserved = gen-delims / sub-delims + + gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + + sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + / "*" / "+" / "," / ";" / "=" + + The purpose of reserved characters is to provide a set of delimiting + characters that are distinguishable from other data within a URI. + URIs that differ in the replacement of a reserved character with its + corresponding percent-encoded octet are not equivalent. Percent- + encoding a reserved character, or decoding a percent-encoded octet + that corresponds to a reserved character, will change how the URI is + interpreted by most applications. Thus, characters in the reserved + set are protected from normalization and are therefore safe to be + used by scheme-specific and producer-specific algorithms for + delimiting data subcomponents within a URI. + + A subset of the reserved characters (gen-delims) is used as + delimiters of the generic URI components described in Section 3. A + component's ABNF syntax rule will not use the reserved or gen-delims + rule names directly; instead, each syntax rule lists the characters + allowed within that component (i.e., not delimiting it), and any of + those characters that are also in the reserved set are "reserved" for + use as subcomponent delimiters within the component. Only the most + common subcomponents are defined by this specification; other + subcomponents may be defined by a URI scheme's specification, or by + the implementation-specific syntax of a URI's dereferencing + algorithm, provided that such subcomponents are delimited by + characters in the reserved set allowed within that component. + + URI producing applications should percent-encode data octets that + correspond to characters in the reserved set unless these characters + are specifically allowed by the URI scheme to represent data in that + component. If a reserved character is found in a URI component and + no delimiting role is known for that character, then it must be + interpreted as representing the data octet corresponding to that + character's encoding in US-ASCII. + +2.3. Unreserved Characters + + Characters that are allowed in a URI but do not have a reserved + purpose are called unreserved. These include uppercase and lowercase + letters, decimal digits, hyphen, period, underscore, and tilde. + + unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + + + + + +Berners-Lee, et al. Standards Track [Page 13] + +RFC 3986 URI Generic Syntax January 2005 + + + URIs that differ in the replacement of an unreserved character with + its corresponding percent-encoded US-ASCII octet are equivalent: they + identify the same resource. However, URI comparison implementations + do not always perform normalization prior to comparison (see Section + 6). For consistency, percent-encoded octets in the ranges of ALPHA + (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), period (%2E), + underscore (%5F), or tilde (%7E) should not be created by URI + producers and, when found in a URI, should be decoded to their + corresponding unreserved characters by URI normalizers. + +2.4. When to Encode or Decode + + Under normal circumstances, the only time when octets within a URI + are percent-encoded is during the process of producing the URI from + its component parts. This is when an implementation determines which + of the reserved characters are to be used as subcomponent delimiters + and which can be safely used as data. Once produced, a URI is always + in its percent-encoded form. + + When a URI is dereferenced, the components and subcomponents + significant to the scheme-specific dereferencing process (if any) + must be parsed and separated before the percent-encoded octets within + those components can be safely decoded, as otherwise the data may be + mistaken for component delimiters. The only exception is for + percent-encoded octets corresponding to characters in the unreserved + set, which can be decoded at any time. For example, the octet + corresponding to the tilde ("~") character is often encoded as "%7E" + by older URI processing implementations; the "%7E" can be replaced by + "~" without changing its interpretation. + + Because the percent ("%") character serves as the indicator for + percent-encoded octets, it must be percent-encoded as "%25" for that + octet to be used as data within a URI. Implementations must not + percent-encode or decode the same string more than once, as decoding + an already decoded string might lead to misinterpreting a percent + data octet as the beginning of a percent-encoding, or vice versa in + the case of percent-encoding an already percent-encoded string. + +2.5. Identifying Data + + URI characters provide identifying data for each of the URI + components, serving as an external interface for identification + between systems. Although the presence and nature of the URI + production interface is hidden from clients that use its URIs (and is + thus beyond the scope of the interoperability requirements defined by + this specification), it is a frequent source of confusion and errors + in the interpretation of URI character issues. Implementers have to + be aware that there are multiple character encodings involved in the + + + +Berners-Lee, et al. Standards Track [Page 14] + +RFC 3986 URI Generic Syntax January 2005 + + + production and transmission of URIs: local name and data encoding, + public interface encoding, URI character encoding, data format + encoding, and protocol encoding. + + Local names, such as file system names, are stored with a local + character encoding. URI producing applications (e.g., origin + servers) will typically use the local encoding as the basis for + producing meaningful names. The URI producer will transform the + local encoding to one that is suitable for a public interface and + then transform the public interface encoding into the restricted set + of URI characters (reserved, unreserved, and percent-encodings). + Those characters are, in turn, encoded as octets to be used as a + reference within a data format (e.g., a document charset), and such + data formats are often subsequently encoded for transmission over + Internet protocols. + + For most systems, an unreserved character appearing within a URI + component is interpreted as representing the data octet corresponding + to that character's encoding in US-ASCII. Consumers of URIs assume + that the letter "X" corresponds to the octet "01011000", and even + when that assumption is incorrect, there is no harm in making it. A + system that internally provides identifiers in the form of a + different character encoding, such as EBCDIC, will generally perform + character translation of textual identifiers to UTF-8 [STD63] (or + some other superset of the US-ASCII character encoding) at an + internal interface, thereby providing more meaningful identifiers + than those resulting from simply percent-encoding the original + octets. + + For example, consider an information service that provides data, + stored locally using an EBCDIC-based file system, to clients on the + Internet through an HTTP server. When an author creates a file with + the name "Laguna Beach" on that file system, the "http" URI + corresponding to that resource is expected to contain the meaningful + string "Laguna%20Beach". If, however, that server produces URIs by + using an overly simplistic raw octet mapping, then the result would + be a URI containing "%D3%81%87%A4%95%81@%C2%85%81%83%88". An + internal transcoding interface fixes this problem by transcoding the + local name to a superset of US-ASCII prior to producing the URI. + Naturally, proper interpretation of an incoming URI on such an + interface requires that percent-encoded octets be decoded (e.g., + "%20" to SP) before the reverse transcoding is applied to obtain the + local name. + + In some cases, the internal interface between a URI component and the + identifying data that it has been crafted to represent is much less + direct than a character encoding translation. For example, portions + of a URI might reflect a query on non-ASCII data, or numeric + + + +Berners-Lee, et al. Standards Track [Page 15] + +RFC 3986 URI Generic Syntax January 2005 + + + coordinates on a map. Likewise, a URI scheme may define components + with additional encoding requirements that are applied prior to + forming the component and producing the URI. + + When a new URI scheme defines a component that represents textual + data consisting of characters from the Universal Character Set [UCS], + the data should first be encoded as octets according to the UTF-8 + character encoding [STD63]; then only those octets that do not + correspond to characters in the unreserved set should be percent- + encoded. For example, the character A would be represented as "A", + the character LATIN CAPITAL LETTER A WITH GRAVE would be represented + as "%C3%80", and the character KATAKANA LETTER A would be represented + as "%E3%82%A2". + +3. Syntax Components + + The generic URI syntax consists of a hierarchical sequence of + components referred to as the scheme, authority, path, query, and + fragment. + + URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] + + hier-part = "//" authority path-abempty + / path-absolute + / path-rootless + / path-empty + + The scheme and path components are required, though the path may be + empty (no characters). When authority is present, the path must + either be empty or begin with a slash ("/") character. When + authority is not present, the path cannot begin with two slash + characters ("//"). These restrictions result in five different ABNF + rules for a path (Section 3.3), only one of which will match any + given URI reference. + + The following are two example URIs and their component parts: + + foo://example.com:8042/over/there?name=ferret#nose + \_/ \______________/\_________/ \_________/ \__/ + | | | | | + scheme authority path query fragment + | _____________________|__ + / \ / \ + urn:example:animal:ferret:nose + + + + + + + +Berners-Lee, et al. Standards Track [Page 16] + +RFC 3986 URI Generic Syntax January 2005 + + +3.1. Scheme + + Each URI begins with a scheme name that refers to a specification for + assigning identifiers within that scheme. As such, the URI syntax is + a federated and extensible naming system wherein each scheme's + specification may further restrict the syntax and semantics of + identifiers using that scheme. + + Scheme names consist of a sequence of characters beginning with a + letter and followed by any combination of letters, digits, plus + ("+"), period ("."), or hyphen ("-"). Although schemes are case- + insensitive, the canonical form is lowercase and documents that + specify schemes must do so with lowercase letters. An implementation + should accept uppercase letters as equivalent to lowercase in scheme + names (e.g., allow "HTTP" as well as "http") for the sake of + robustness but should only produce lowercase scheme names for + consistency. + + scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + + Individual schemes are not specified by this document. The process + for registration of new URI schemes is defined separately by [BCP35]. + The scheme registry maintains the mapping between scheme names and + their specifications. Advice for designers of new URI schemes can be + found in [RFC2718]. URI scheme specifications must define their own + syntax so that all strings matching their scheme-specific syntax will + also match the <absolute-URI> grammar, as described in Section 4.3. + + When presented with a URI that violates one or more scheme-specific + restrictions, the scheme-specific resolution process should flag the + reference as an error rather than ignore the unused parts; doing so + reduces the number of equivalent URIs and helps detect abuses of the + generic syntax, which might indicate that the URI has been + constructed to mislead the user (Section 7.6). + +3.2. Authority + + Many URI schemes include a hierarchical element for a naming + authority so that governance of the name space defined by the + remainder of the URI is delegated to that authority (which may, in + turn, delegate it further). The generic syntax provides a common + means for distinguishing an authority based on a registered name or + server address, along with optional port and user information. + + The authority component is preceded by a double slash ("//") and is + terminated by the next slash ("/"), question mark ("?"), or number + sign ("#") character, or by the end of the URI. + + + + +Berners-Lee, et al. Standards Track [Page 17] + +RFC 3986 URI Generic Syntax January 2005 + + + authority = [ userinfo "@" ] host [ ":" port ] + + URI producers and normalizers should omit the ":" delimiter that + separates host from port if the port component is empty. Some + schemes do not allow the userinfo and/or port subcomponents. + + If a URI contains an authority component, then the path component + must either be empty or begin with a slash ("/") character. Non- + validating parsers (those that merely separate a URI reference into + its major components) will often ignore the subcomponent structure of + authority, treating it as an opaque string from the double-slash to + the first terminating delimiter, until such time as the URI is + dereferenced. + +3.2.1. User Information + + The userinfo subcomponent may consist of a user name and, optionally, + scheme-specific information about how to gain authorization to access + the resource. The user information, if present, is followed by a + commercial at-sign ("@") that delimits it from the host. + + userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + + Use of the format "user:password" in the userinfo field is + deprecated. Applications should not render as clear text any data + after the first colon (":") character found within a userinfo + subcomponent unless the data after the colon is the empty string + (indicating no password). Applications may choose to ignore or + reject such data when it is received as part of a reference and + should reject the storage of such data in unencrypted form. The + passing of authentication information in clear text has proven to be + a security risk in almost every case where it has been used. + + Applications that render a URI for the sake of user feedback, such as + in graphical hypertext browsing, should render userinfo in a way that + is distinguished from the rest of a URI, when feasible. Such + rendering will assist the user in cases where the userinfo has been + misleadingly crafted to look like a trusted domain name + (Section 7.6). + +3.2.2. Host + + The host subcomponent of authority is identified by an IP literal + encapsulated within square brackets, an IPv4 address in dotted- + decimal form, or a registered name. The host subcomponent is case- + insensitive. The presence of a host subcomponent within a URI does + not imply that the scheme requires access to the given host on the + Internet. In many cases, the host syntax is used only for the sake + + + +Berners-Lee, et al. Standards Track [Page 18] + +RFC 3986 URI Generic Syntax January 2005 + + + of reusing the existing registration process created and deployed for + DNS, thus obtaining a globally unique name without the cost of + deploying another registry. However, such use comes with its own + costs: domain name ownership may change over time for reasons not + anticipated by the URI producer. In other cases, the data within the + host component identifies a registered name that has nothing to do + with an Internet host. We use the name "host" for the ABNF rule + because that is its most common purpose, not its only purpose. + + host = IP-literal / IPv4address / reg-name + + The syntax rule for host is ambiguous because it does not completely + distinguish between an IPv4address and a reg-name. In order to + disambiguate the syntax, we apply the "first-match-wins" algorithm: + If host matches the rule for IPv4address, then it should be + considered an IPv4 address literal and not a reg-name. Although host + is case-insensitive, producers and normalizers should use lowercase + for registered names and hexadecimal addresses for the sake of + uniformity, while only using uppercase letters for percent-encodings. + + A host identified by an Internet Protocol literal address, version 6 + [RFC3513] or later, is distinguished by enclosing the IP literal + within square brackets ("[" and "]"). This is the only place where + square bracket characters are allowed in the URI syntax. In + anticipation of future, as-yet-undefined IP literal address formats, + an implementation may use an optional version flag to indicate such a + format explicitly rather than rely on heuristic determination. + + IP-literal = "[" ( IPv6address / IPvFuture ) "]" + + IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) + + The version flag does not indicate the IP version; rather, it + indicates future versions of the literal format. As such, + implementations must not provide the version flag for the existing + IPv4 and IPv6 literal address forms described below. If a URI + containing an IP-literal that starts with "v" (case-insensitive), + indicating that the version flag is present, is dereferenced by an + application that does not know the meaning of that version flag, then + the application should return an appropriate error for "address + mechanism not supported". + + A host identified by an IPv6 literal address is represented inside + the square brackets without a preceding version flag. The ABNF + provided here is a translation of the text definition of an IPv6 + literal address provided in [RFC3513]. This syntax does not support + IPv6 scoped addressing zone identifiers. + + + + +Berners-Lee, et al. Standards Track [Page 19] + +RFC 3986 URI Generic Syntax January 2005 + + + A 128-bit IPv6 address is divided into eight 16-bit pieces. Each + piece is represented numerically in case-insensitive hexadecimal, + using one to four hexadecimal digits (leading zeroes are permitted). + The eight encoded pieces are given most-significant first, separated + by colon characters. Optionally, the least-significant two pieces + may instead be represented in IPv4 address textual format. A + sequence of one or more consecutive zero-valued 16-bit pieces within + the address may be elided, omitting all their digits and leaving + exactly two consecutive colons in their place to mark the elision. + + IPv6address = 6( h16 ":" ) ls32 + / "::" 5( h16 ":" ) ls32 + / [ h16 ] "::" 4( h16 ":" ) ls32 + / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + / [ *4( h16 ":" ) h16 ] "::" ls32 + / [ *5( h16 ":" ) h16 ] "::" h16 + / [ *6( h16 ":" ) h16 ] "::" + + ls32 = ( h16 ":" h16 ) / IPv4address + ; least-significant 32 bits of address + + h16 = 1*4HEXDIG + ; 16 bits of address represented in hexadecimal + + A host identified by an IPv4 literal address is represented in + dotted-decimal notation (a sequence of four decimal numbers in the + range 0 to 255, separated by "."), as described in [RFC1123] by + reference to [RFC0952]. Note that other forms of dotted notation may + be interpreted on some platforms, as described in Section 7.4, but + only the dotted-decimal form of four octets is allowed by this + grammar. + + IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + + dec-octet = DIGIT ; 0-9 + / %x31-39 DIGIT ; 10-99 + / "1" 2DIGIT ; 100-199 + / "2" %x30-34 DIGIT ; 200-249 + / "25" %x30-35 ; 250-255 + + A host identified by a registered name is a sequence of characters + usually intended for lookup within a locally defined host or service + name registry, though the URI's scheme-specific semantics may require + that a specific registry (or fixed name table) be used instead. The + most common name registry mechanism is the Domain Name System (DNS). + A registered name intended for lookup in the DNS uses the syntax + + + +Berners-Lee, et al. Standards Track [Page 20] + +RFC 3986 URI Generic Syntax January 2005 + + + defined in Section 3.5 of [RFC1034] and Section 2.1 of [RFC1123]. + Such a name consists of a sequence of domain labels separated by ".", + each domain label starting and ending with an alphanumeric character + and possibly also containing "-" characters. The rightmost domain + label of a fully qualified domain name in DNS may be followed by a + single "." and should be if it is necessary to distinguish between + the complete domain name and some local domain. + + reg-name = *( unreserved / pct-encoded / sub-delims ) + + If the URI scheme defines a default for host, then that default + applies when the host subcomponent is undefined or when the + registered name is empty (zero length). For example, the "file" URI + scheme is defined so that no authority, an empty host, and + "localhost" all mean the end-user's machine, whereas the "http" + scheme considers a missing authority or empty host invalid. + + This specification does not mandate a particular registered name + lookup technology and therefore does not restrict the syntax of reg- + name beyond what is necessary for interoperability. Instead, it + delegates the issue of registered name syntax conformance to the + operating system of each application performing URI resolution, and + that operating system decides what it will allow for the purpose of + host identification. A URI resolution implementation might use DNS, + host tables, yellow pages, NetInfo, WINS, or any other system for + lookup of registered names. However, a globally scoped naming + system, such as DNS fully qualified domain names, is necessary for + URIs intended to have global scope. URI producers should use names + that conform to the DNS syntax, even when use of DNS is not + immediately apparent, and should limit these names to no more than + 255 characters in length. + + The reg-name syntax allows percent-encoded octets in order to + represent non-ASCII registered names in a uniform way that is + independent of the underlying name resolution technology. Non-ASCII + characters must first be encoded according to UTF-8 [STD63], and then + each octet of the corresponding UTF-8 sequence must be percent- + encoded to be represented as URI characters. URI producing + applications must not use percent-encoding in host unless it is used + to represent a UTF-8 character sequence. When a non-ASCII registered + name represents an internationalized domain name intended for + resolution via the DNS, the name must be transformed to the IDNA + encoding [RFC3490] prior to name lookup. URI producers should + provide these registered names in the IDNA encoding, rather than a + percent-encoding, if they wish to maximize interoperability with + legacy URI resolvers. + + + + + +Berners-Lee, et al. Standards Track [Page 21] + +RFC 3986 URI Generic Syntax January 2005 + + +3.2.3. Port + + The port subcomponent of authority is designated by an optional port + number in decimal following the host and delimited from it by a + single colon (":") character. + + port = *DIGIT + + A scheme may define a default port. For example, the "http" scheme + defines a default port of "80", corresponding to its reserved TCP + port number. The type of port designated by the port number (e.g., + TCP, UDP, SCTP) is defined by the URI scheme. URI producers and + normalizers should omit the port component and its ":" delimiter if + port is empty or if its value would be the same as that of the + scheme's default. + +3.3. Path + + The path component contains data, usually organized in hierarchical + form, that, along with data in the non-hierarchical query component + (Section 3.4), serves to identify a resource within the scope of the + URI's scheme and naming authority (if any). The path is terminated + by the first question mark ("?") or number sign ("#") character, or + by the end of the URI. + + If a URI contains an authority component, then the path component + must either be empty or begin with a slash ("/") character. If a URI + does not contain an authority component, then the path cannot begin + with two slash characters ("//"). In addition, a URI reference + (Section 4.1) may be a relative-path reference, in which case the + first path segment cannot contain a colon (":") character. The ABNF + requires five separate rules to disambiguate these cases, only one of + which will match the path substring within a given URI reference. We + use the generic term "path component" to describe the URI substring + matched by the parser to one of these rules. + + path = path-abempty ; begins with "/" or is empty + / path-absolute ; begins with "/" but not "//" + / path-noscheme ; begins with a non-colon segment + / path-rootless ; begins with a segment + / path-empty ; zero characters + + path-abempty = *( "/" segment ) + path-absolute = "/" [ segment-nz *( "/" segment ) ] + path-noscheme = segment-nz-nc *( "/" segment ) + path-rootless = segment-nz *( "/" segment ) + path-empty = 0<pchar> + + + + +Berners-Lee, et al. Standards Track [Page 22] + +RFC 3986 URI Generic Syntax January 2005 + + + segment = *pchar + segment-nz = 1*pchar + segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + ; non-zero-length segment without any colon ":" + + pchar = unreserved / pct-encoded / sub-delims / ":" / "@" + + A path consists of a sequence of path segments separated by a slash + ("/") character. A path is always defined for a URI, though the + defined path may be empty (zero length). Use of the slash character + to indicate hierarchy is only required when a URI will be used as the + context for relative references. For example, the URI + <mailto:[email protected]> has a path of "[email protected]", whereas + the URI <foo://info.example.com?fred> has an empty path. + + The path segments "." and "..", also known as dot-segments, are + defined for relative reference within the path name hierarchy. They + are intended for use at the beginning of a relative-path reference + (Section 4.2) to indicate relative position within the hierarchical + tree of names. This is similar to their role within some operating + systems' file directory structures to indicate the current directory + and parent directory, respectively. However, unlike in a file + system, these dot-segments are only interpreted within the URI path + hierarchy and are removed as part of the resolution process (Section + 5.2). + + Aside from dot-segments in hierarchical paths, a path segment is + considered opaque by the generic syntax. URI producing applications + often use the reserved characters allowed in a segment to delimit + scheme-specific or dereference-handler-specific subcomponents. For + example, the semicolon (";") and equals ("=") reserved characters are + often used to delimit parameters and parameter values applicable to + that segment. The comma (",") reserved character is often used for + similar purposes. For example, one URI producer might use a segment + such as "name;v=1.1" to indicate a reference to version 1.1 of + "name", whereas another might use a segment such as "name,1.1" to + indicate the same. Parameter types may be defined by scheme-specific + semantics, but in most cases the syntax of a parameter is specific to + the implementation of the URI's dereferencing algorithm. + +3.4. Query + + The query component contains non-hierarchical data that, along with + data in the path component (Section 3.3), serves to identify a + resource within the scope of the URI's scheme and naming authority + (if any). The query component is indicated by the first question + mark ("?") character and terminated by a number sign ("#") character + or by the end of the URI. + + + +Berners-Lee, et al. Standards Track [Page 23] + +RFC 3986 URI Generic Syntax January 2005 + + + query = *( pchar / "/" / "?" ) + + The characters slash ("/") and question mark ("?") may represent data + within the query component. Beware that some older, erroneous + implementations may not handle such data correctly when it is used as + the base URI for relative references (Section 5.1), apparently + because they fail to distinguish query data from path data when + looking for hierarchical separators. However, as query components + are often used to carry identifying information in the form of + "key=value" pairs and one frequently used value is a reference to + another URI, it is sometimes better for usability to avoid percent- + encoding those characters. + +3.5. Fragment + + The fragment identifier component of a URI allows indirect + identification of a secondary resource by reference to a primary + resource and additional identifying information. The identified + secondary resource may be some portion or subset of the primary + resource, some view on representations of the primary resource, or + some other resource defined or described by those representations. A + fragment identifier component is indicated by the presence of a + number sign ("#") character and terminated by the end of the URI. + + fragment = *( pchar / "/" / "?" ) + + The semantics of a fragment identifier are defined by the set of + representations that might result from a retrieval action on the + primary resource. The fragment's format and resolution is therefore + dependent on the media type [RFC2046] of a potentially retrieved + representation, even though such a retrieval is only performed if the + URI is dereferenced. If no such representation exists, then the + semantics of the fragment are considered unknown and are effectively + unconstrained. Fragment identifier semantics are independent of the + URI scheme and thus cannot be redefined by scheme specifications. + + Individual media types may define their own restrictions on or + structures within the fragment identifier syntax for specifying + different types of subsets, views, or external references that are + identifiable as secondary resources by that media type. If the + primary resource has multiple representations, as is often the case + for resources whose representation is selected based on attributes of + the retrieval request (a.k.a., content negotiation), then whatever is + identified by the fragment should be consistent across all of those + representations. Each representation should either define the + fragment so that it corresponds to the same secondary resource, + regardless of how it is represented, or should leave the fragment + undefined (i.e., not found). + + + +Berners-Lee, et al. Standards Track [Page 24] + +RFC 3986 URI Generic Syntax January 2005 + + + As with any URI, use of a fragment identifier component does not + imply that a retrieval action will take place. A URI with a fragment + identifier may be used to refer to the secondary resource without any + implication that the primary resource is accessible or will ever be + accessed. + + Fragment identifiers have a special role in information retrieval + systems as the primary form of client-side indirect referencing, + allowing an author to specifically identify aspects of an existing + resource that are only indirectly provided by the resource owner. As + such, the fragment identifier is not used in the scheme-specific + processing of a URI; instead, the fragment identifier is separated + from the rest of the URI prior to a dereference, and thus the + identifying information within the fragment itself is dereferenced + solely by the user agent, regardless of the URI scheme. Although + this separate handling is often perceived to be a loss of + information, particularly for accurate redirection of references as + resources move over time, it also serves to prevent information + providers from denying reference authors the right to refer to + information within a resource selectively. Indirect referencing also + provides additional flexibility and extensibility to systems that use + URIs, as new media types are easier to define and deploy than new + schemes of identification. + + The characters slash ("/") and question mark ("?") are allowed to + represent data within the fragment identifier. Beware that some + older, erroneous implementations may not handle this data correctly + when it is used as the base URI for relative references (Section + 5.1). + +4. Usage + + When applications make reference to a URI, they do not always use the + full form of reference defined by the "URI" syntax rule. To save + space and take advantage of hierarchical locality, many Internet + protocol elements and media type formats allow an abbreviation of a + URI, whereas others restrict the syntax to a particular form of URI. + We define the most common forms of reference syntax in this + specification because they impact and depend upon the design of the + generic syntax, requiring a uniform parsing algorithm in order to be + interpreted consistently. + +4.1. URI Reference + + URI-reference is used to denote the most common usage of a resource + identifier. + + URI-reference = URI / relative-ref + + + +Berners-Lee, et al. Standards Track [Page 25] + +RFC 3986 URI Generic Syntax January 2005 + + + A URI-reference is either a URI or a relative reference. If the + URI-reference's prefix does not match the syntax of a scheme followed + by its colon separator, then the URI-reference is a relative + reference. + + A URI-reference is typically parsed first into the five URI + components, in order to determine what components are present and + whether the reference is relative. Then, each component is parsed + for its subparts and their validation. The ABNF of URI-reference, + along with the "first-match-wins" disambiguation rule, is sufficient + to define a validating parser for the generic syntax. Readers + familiar with regular expressions should see Appendix B for an + example of a non-validating URI-reference parser that will take any + given string and extract the URI components. + +4.2. Relative Reference + + A relative reference takes advantage of the hierarchical syntax + (Section 1.2.3) to express a URI reference relative to the name space + of another hierarchical URI. + + relative-ref = relative-part [ "?" query ] [ "#" fragment ] + + relative-part = "//" authority path-abempty + / path-absolute + / path-noscheme + / path-empty + + The URI referred to by a relative reference, also known as the target + URI, is obtained by applying the reference resolution algorithm of + Section 5. + + A relative reference that begins with two slash characters is termed + a network-path reference; such references are rarely used. A + relative reference that begins with a single slash character is + termed an absolute-path reference. A relative reference that does + not begin with a slash character is termed a relative-path reference. + + A path segment that contains a colon character (e.g., "this:that") + cannot be used as the first segment of a relative-path reference, as + it would be mistaken for a scheme name. Such a segment must be + preceded by a dot-segment (e.g., "./this:that") to make a relative- + path reference. + + + + + + + + +Berners-Lee, et al. Standards Track [Page 26] + +RFC 3986 URI Generic Syntax January 2005 + + +4.3. Absolute URI + + Some protocol elements allow only the absolute form of a URI without + a fragment identifier. For example, defining a base URI for later + use by relative references calls for an absolute-URI syntax rule that + does not allow a fragment. + + absolute-URI = scheme ":" hier-part [ "?" query ] + + URI scheme specifications must define their own syntax so that all + strings matching their scheme-specific syntax will also match the + <absolute-URI> grammar. Scheme specifications will not define + fragment identifier syntax or usage, regardless of its applicability + to resources identifiable via that scheme, as fragment identification + is orthogonal to scheme definition. However, scheme specifications + are encouraged to include a wide range of examples, including + examples that show use of the scheme's URIs with fragment identifiers + when such usage is appropriate. + +4.4. Same-Document Reference + + When a URI reference refers to a URI that is, aside from its fragment + component (if any), identical to the base URI (Section 5.1), that + reference is called a "same-document" reference. The most frequent + examples of same-document references are relative references that are + empty or include only the number sign ("#") separator followed by a + fragment identifier. + + When a same-document reference is dereferenced for a retrieval + action, the target of that reference is defined to be within the same + entity (representation, document, or message) as the reference; + therefore, a dereference should not result in a new retrieval action. + + Normalization of the base and target URIs prior to their comparison, + as described in Sections 6.2.2 and 6.2.3, is allowed but rarely + performed in practice. Normalization may increase the set of same- + document references, which may be of benefit to some caching + applications. As such, reference authors should not assume that a + slightly different, though equivalent, reference URI will (or will + not) be interpreted as a same-document reference by any given + application. + +4.5. Suffix Reference + + The URI syntax is designed for unambiguous reference to resources and + extensibility via the URI scheme. However, as URI identification and + usage have become commonplace, traditional media (television, radio, + newspapers, billboards, etc.) have increasingly used a suffix of the + + + +Berners-Lee, et al. Standards Track [Page 27] + +RFC 3986 URI Generic Syntax January 2005 + + + URI as a reference, consisting of only the authority and path + portions of the URI, such as + + www.w3.org/Addressing/ + + or simply a DNS registered name on its own. Such references are + primarily intended for human interpretation rather than for machines, + with the assumption that context-based heuristics are sufficient to + complete the URI (e.g., most registered names beginning with "www" + are likely to have a URI prefix of "http://"). Although there is no + standard set of heuristics for disambiguating a URI suffix, many + client implementations allow them to be entered by the user and + heuristically resolved. + + Although this practice of using suffix references is common, it + should be avoided whenever possible and should never be used in + situations where long-term references are expected. The heuristics + noted above will change over time, particularly when a new URI scheme + becomes popular, and are often incorrect when used out of context. + Furthermore, they can lead to security issues along the lines of + those described in [RFC1535]. + + As a URI suffix has the same syntax as a relative-path reference, a + suffix reference cannot be used in contexts where a relative + reference is expected. As a result, suffix references are limited to + places where there is no defined base URI, such as dialog boxes and + off-line advertisements. + +5. Reference Resolution + + This section defines the process of resolving a URI reference within + a context that allows relative references so that the result is a + string matching the <URI> syntax rule of Section 3. + +5.1. Establishing a Base URI + + The term "relative" implies that a "base URI" exists against which + the relative reference is applied. Aside from fragment-only + references (Section 4.4), relative references are only usable when a + base URI is known. A base URI must be established by the parser + prior to parsing URI references that might be relative. A base URI + must conform to the <absolute-URI> syntax rule (Section 4.3). If the + base URI is obtained from a URI reference, then that reference must + be converted to absolute form and stripped of any fragment component + prior to its use as a base URI. + + + + + + +Berners-Lee, et al. Standards Track [Page 28] + +RFC 3986 URI Generic Syntax January 2005 + + + The base URI of a reference can be established in one of four ways, + discussed below in order of precedence. The order of precedence can + be thought of in terms of layers, where the innermost defined base + URI has the highest precedence. This can be visualized graphically + as follows: + + .----------------------------------------------------------. + | .----------------------------------------------------. | + | | .----------------------------------------------. | | + | | | .----------------------------------------. | | | + | | | | .----------------------------------. | | | | + | | | | | <relative-reference> | | | | | + | | | | `----------------------------------' | | | | + | | | | (5.1.1) Base URI embedded in content | | | | + | | | `----------------------------------------' | | | + | | | (5.1.2) Base URI of the encapsulating entity | | | + | | | (message, representation, or none) | | | + | | `----------------------------------------------' | | + | | (5.1.3) URI used to retrieve the entity | | + | `----------------------------------------------------' | + | (5.1.4) Default Base URI (application-dependent) | + `----------------------------------------------------------' + +5.1.1. Base URI Embedded in Content + + Within certain media types, a base URI for relative references can be + embedded within the content itself so that it can be readily obtained + by a parser. This can be useful for descriptive documents, such as + tables of contents, which may be transmitted to others through + protocols other than their usual retrieval context (e.g., email or + USENET news). + + It is beyond the scope of this specification to specify how, for each + media type, a base URI can be embedded. The appropriate syntax, when + available, is described by the data format specification associated + with each media type. + +5.1.2. Base URI from the Encapsulating Entity + + If no base URI is embedded, the base URI is defined by the + representation's retrieval context. For a document that is enclosed + within another entity, such as a message or archive, the retrieval + context is that entity. Thus, the default base URI of a + representation is the base URI of the entity in which the + representation is encapsulated. + + + + + + +Berners-Lee, et al. Standards Track [Page 29] + +RFC 3986 URI Generic Syntax January 2005 + + + A mechanism for embedding a base URI within MIME container types + (e.g., the message and multipart types) is defined by MHTML + [RFC2557]. Protocols that do not use the MIME message header syntax, + but that do allow some form of tagged metadata to be included within + messages, may define their own syntax for defining a base URI as part + of a message. + +5.1.3. Base URI from the Retrieval URI + + If no base URI is embedded and the representation is not encapsulated + within some other entity, then, if a URI was used to retrieve the + representation, that URI shall be considered the base URI. Note that + if the retrieval was the result of a redirected request, the last URI + used (i.e., the URI that resulted in the actual retrieval of the + representation) is the base URI. + +5.1.4. Default Base URI + + If none of the conditions described above apply, then the base URI is + defined by the context of the application. As this definition is + necessarily application-dependent, failing to define a base URI by + using one of the other methods may result in the same content being + interpreted differently by different types of applications. + + A sender of a representation containing relative references is + responsible for ensuring that a base URI for those references can be + established. Aside from fragment-only references, relative + references can only be used reliably in situations where the base URI + is well defined. + +5.2. Relative Resolution + + This section describes an algorithm for converting a URI reference + that might be relative to a given base URI into the parsed components + of the reference's target. The components can then be recomposed, as + described in Section 5.3, to form the target URI. This algorithm + provides definitive results that can be used to test the output of + other implementations. Applications may implement relative reference + resolution by using some other algorithm, provided that the results + match what would be given by this one. + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 30] + +RFC 3986 URI Generic Syntax January 2005 + + +5.2.1. Pre-parse the Base URI + + The base URI (Base) is established according to the procedure of + Section 5.1 and parsed into the five main components described in + Section 3. Note that only the scheme component is required to be + present in a base URI; the other components may be empty or + undefined. A component is undefined if its associated delimiter does + not appear in the URI reference; the path component is never + undefined, though it may be empty. + + Normalization of the base URI, as described in Sections 6.2.2 and + 6.2.3, is optional. A URI reference must be transformed to its + target URI before it can be normalized. + +5.2.2. Transform References + + For each URI reference (R), the following pseudocode describes an + algorithm for transforming R into its target URI (T): + + -- The URI reference is parsed into the five URI components + -- + (R.scheme, R.authority, R.path, R.query, R.fragment) = parse(R); + + -- A non-strict parser may ignore a scheme in the reference + -- if it is identical to the base URI's scheme. + -- + if ((not strict) and (R.scheme == Base.scheme)) then + undefine(R.scheme); + endif; + + + + + + + + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 31] + +RFC 3986 URI Generic Syntax January 2005 + + + if defined(R.scheme) then + T.scheme = R.scheme; + T.authority = R.authority; + T.path = remove_dot_segments(R.path); + T.query = R.query; + else + if defined(R.authority) then + T.authority = R.authority; + T.path = remove_dot_segments(R.path); + T.query = R.query; + else + if (R.path == "") then + T.path = Base.path; + if defined(R.query) then + T.query = R.query; + else + T.query = Base.query; + endif; + else + if (R.path starts-with "/") then + T.path = remove_dot_segments(R.path); + else + T.path = merge(Base.path, R.path); + T.path = remove_dot_segments(T.path); + endif; + T.query = R.query; + endif; + T.authority = Base.authority; + endif; + T.scheme = Base.scheme; + endif; + + T.fragment = R.fragment; + +5.2.3. Merge Paths + + The pseudocode above refers to a "merge" routine for merging a + relative-path reference with the path of the base URI. This is + accomplished as follows: + + o If the base URI has a defined authority component and an empty + path, then return a string consisting of "/" concatenated with the + reference's path; otherwise, + + + + + + + + +Berners-Lee, et al. Standards Track [Page 32] + +RFC 3986 URI Generic Syntax January 2005 + + + o return a string consisting of the reference's path component + appended to all but the last segment of the base URI's path (i.e., + excluding any characters after the right-most "/" in the base URI + path, or excluding the entire base URI path if it does not contain + any "/" characters). + +5.2.4. Remove Dot Segments + + The pseudocode also refers to a "remove_dot_segments" routine for + interpreting and removing the special "." and ".." complete path + segments from a referenced path. This is done after the path is + extracted from a reference, whether or not the path was relative, in + order to remove any invalid or extraneous dot-segments prior to + forming the target URI. Although there are many ways to accomplish + this removal process, we describe a simple method using two string + buffers. + + 1. The input buffer is initialized with the now-appended path + components and the output buffer is initialized to the empty + string. + + 2. While the input buffer is not empty, loop as follows: + + A. If the input buffer begins with a prefix of "../" or "./", + then remove that prefix from the input buffer; otherwise, + + B. if the input buffer begins with a prefix of "/./" or "/.", + where "." is a complete path segment, then replace that + prefix with "/" in the input buffer; otherwise, + + C. if the input buffer begins with a prefix of "/../" or "/..", + where ".." is a complete path segment, then replace that + prefix with "/" in the input buffer and remove the last + segment and its preceding "/" (if any) from the output + buffer; otherwise, + + D. if the input buffer consists only of "." or "..", then remove + that from the input buffer; otherwise, + + E. move the first path segment in the input buffer to the end of + the output buffer, including the initial "/" character (if + any) and any subsequent characters up to, but not including, + the next "/" character or the end of the input buffer. + + 3. Finally, the output buffer is returned as the result of + remove_dot_segments. + + + + + +Berners-Lee, et al. Standards Track [Page 33] + +RFC 3986 URI Generic Syntax January 2005 + + + Note that dot-segments are intended for use in URI references to + express an identifier relative to the hierarchy of names in the base + URI. The remove_dot_segments algorithm respects that hierarchy by + removing extra dot-segments rather than treat them as an error or + leaving them to be misinterpreted by dereference implementations. + + The following illustrates how the above steps are applied for two + examples of merged paths, showing the state of the two buffers after + each step. + + STEP OUTPUT BUFFER INPUT BUFFER + + 1 : /a/b/c/./../../g + 2E: /a /b/c/./../../g + 2E: /a/b /c/./../../g + 2E: /a/b/c /./../../g + 2B: /a/b/c /../../g + 2C: /a/b /../g + 2C: /a /g + 2E: /a/g + + STEP OUTPUT BUFFER INPUT BUFFER + + 1 : mid/content=5/../6 + 2E: mid /content=5/../6 + 2E: mid/content=5 /../6 + 2C: mid /6 + 2E: mid/6 + + Some applications may find it more efficient to implement the + remove_dot_segments algorithm by using two segment stacks rather than + strings. + + Note: Beware that some older, erroneous implementations will fail + to separate a reference's query component from its path component + prior to merging the base and reference paths, resulting in an + interoperability failure if the query component contains the + strings "/../" or "/./". + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 34] + +RFC 3986 URI Generic Syntax January 2005 + + +5.3. Component Recomposition + + Parsed URI components can be recomposed to obtain the corresponding + URI reference string. Using pseudocode, this would be: + + result = "" + + if defined(scheme) then + append scheme to result; + append ":" to result; + endif; + + if defined(authority) then + append "//" to result; + append authority to result; + endif; + + append path to result; + + if defined(query) then + append "?" to result; + append query to result; + endif; + + if defined(fragment) then + append "#" to result; + append fragment to result; + endif; + + return result; + + Note that we are careful to preserve the distinction between a + component that is undefined, meaning that its separator was not + present in the reference, and a component that is empty, meaning that + the separator was present and was immediately followed by the next + component separator or the end of the reference. + +5.4. Reference Resolution Examples + + Within a representation with a well defined base URI of + + http://a/b/c/d;p?q + + a relative reference is transformed to its target URI as follows. + + + + + + + +Berners-Lee, et al. Standards Track [Page 35] + +RFC 3986 URI Generic Syntax January 2005 + + +5.4.1. Normal Examples + + "g:h" = "g:h" + "g" = "http://a/b/c/g" + "./g" = "http://a/b/c/g" + "g/" = "http://a/b/c/g/" + "/g" = "http://a/g" + "//g" = "http://g" + "?y" = "http://a/b/c/d;p?y" + "g?y" = "http://a/b/c/g?y" + "#s" = "http://a/b/c/d;p?q#s" + "g#s" = "http://a/b/c/g#s" + "g?y#s" = "http://a/b/c/g?y#s" + ";x" = "http://a/b/c/;x" + "g;x" = "http://a/b/c/g;x" + "g;x?y#s" = "http://a/b/c/g;x?y#s" + "" = "http://a/b/c/d;p?q" + "." = "http://a/b/c/" + "./" = "http://a/b/c/" + ".." = "http://a/b/" + "../" = "http://a/b/" + "../g" = "http://a/b/g" + "../.." = "http://a/" + "../../" = "http://a/" + "../../g" = "http://a/g" + +5.4.2. Abnormal Examples + + Although the following abnormal examples are unlikely to occur in + normal practice, all URI parsers should be capable of resolving them + consistently. Each example uses the same base as that above. + + Parsers must be careful in handling cases where there are more ".." + segments in a relative-path reference than there are hierarchical + levels in the base URI's path. Note that the ".." syntax cannot be + used to change the authority component of a URI. + + "../../../g" = "http://a/g" + "../../../../g" = "http://a/g" + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 36] + +RFC 3986 URI Generic Syntax January 2005 + + + Similarly, parsers must remove the dot-segments "." and ".." when + they are complete components of a path, but not when they are only + part of a segment. + + "/./g" = "http://a/g" + "/../g" = "http://a/g" + "g." = "http://a/b/c/g." + ".g" = "http://a/b/c/.g" + "g.." = "http://a/b/c/g.." + "..g" = "http://a/b/c/..g" + + Less likely are cases where the relative reference uses unnecessary + or nonsensical forms of the "." and ".." complete path segments. + + "./../g" = "http://a/b/g" + "./g/." = "http://a/b/c/g/" + "g/./h" = "http://a/b/c/g/h" + "g/../h" = "http://a/b/c/h" + "g;x=1/./y" = "http://a/b/c/g;x=1/y" + "g;x=1/../y" = "http://a/b/c/y" + + Some applications fail to separate the reference's query and/or + fragment components from the path component before merging it with + the base path and removing dot-segments. This error is rarely + noticed, as typical usage of a fragment never includes the hierarchy + ("/") character and the query component is not normally used within + relative references. + + "g?y/./x" = "http://a/b/c/g?y/./x" + "g?y/../x" = "http://a/b/c/g?y/../x" + "g#s/./x" = "http://a/b/c/g#s/./x" + "g#s/../x" = "http://a/b/c/g#s/../x" + + Some parsers allow the scheme name to be present in a relative + reference if it is the same as the base URI scheme. This is + considered to be a loophole in prior specifications of partial URI + [RFC1630]. Its use should be avoided but is allowed for backward + compatibility. + + "http:g" = "http:g" ; for strict parsers + / "http://a/b/c/g" ; for backward compatibility + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 37] + +RFC 3986 URI Generic Syntax January 2005 + + +6. Normalization and Comparison + + One of the most common operations on URIs is simple comparison: + determining whether two URIs are equivalent without using the URIs to + access their respective resource(s). A comparison is performed every + time a response cache is accessed, a browser checks its history to + color a link, or an XML parser processes tags within a namespace. + Extensive normalization prior to comparison of URIs is often used by + spiders and indexing engines to prune a search space or to reduce + duplication of request actions and response storage. + + URI comparison is performed for some particular purpose. Protocols + or implementations that compare URIs for different purposes will + often be subject to differing design trade-offs in regards to how + much effort should be spent in reducing aliased identifiers. This + section describes various methods that may be used to compare URIs, + the trade-offs between them, and the types of applications that might + use them. + +6.1. Equivalence + + Because URIs exist to identify resources, presumably they should be + considered equivalent when they identify the same resource. However, + this definition of equivalence is not of much practical use, as there + is no way for an implementation to compare two resources unless it + has full knowledge or control of them. For this reason, + determination of equivalence or difference of URIs is based on string + comparison, perhaps augmented by reference to additional rules + provided by URI scheme definitions. We use the terms "different" and + "equivalent" to describe the possible outcomes of such comparisons, + but there are many application-dependent versions of equivalence. + + Even though it is possible to determine that two URIs are equivalent, + URI comparison is not sufficient to determine whether two URIs + identify different resources. For example, an owner of two different + domain names could decide to serve the same resource from both, + resulting in two different URIs. Therefore, comparison methods are + designed to minimize false negatives while strictly avoiding false + positives. + + In testing for equivalence, applications should not directly compare + relative references; the references should be converted to their + respective target URIs before comparison. When URIs are compared to + select (or avoid) a network action, such as retrieval of a + representation, fragment components (if any) should be excluded from + the comparison. + + + + + +Berners-Lee, et al. Standards Track [Page 38] + +RFC 3986 URI Generic Syntax January 2005 + + +6.2. Comparison Ladder + + A variety of methods are used in practice to test URI equivalence. + These methods fall into a range, distinguished by the amount of + processing required and the degree to which the probability of false + negatives is reduced. As noted above, false negatives cannot be + eliminated. In practice, their probability can be reduced, but this + reduction requires more processing and is not cost-effective for all + applications. + + If this range of comparison practices is considered as a ladder, the + following discussion will climb the ladder, starting with practices + that are cheap but have a relatively higher chance of producing false + negatives, and proceeding to those that have higher computational + cost and lower risk of false negatives. + +6.2.1. Simple String Comparison + + If two URIs, when considered as character strings, are identical, + then it is safe to conclude that they are equivalent. This type of + equivalence test has very low computational cost and is in wide use + in a variety of applications, particularly in the domain of parsing. + + Testing strings for equivalence requires some basic precautions. + This procedure is often referred to as "bit-for-bit" or + "byte-for-byte" comparison, which is potentially misleading. Testing + strings for equality is normally based on pair comparison of the + characters that make up the strings, starting from the first and + proceeding until both strings are exhausted and all characters are + found to be equal, until a pair of characters compares unequal, or + until one of the strings is exhausted before the other. + + This character comparison requires that each pair of characters be + put in comparable form. For example, should one URI be stored in a + byte array in EBCDIC encoding and the second in a Java String object + (UTF-16), bit-for-bit comparisons applied naively will produce + errors. It is better to speak of equality on a character-for- + character basis rather than on a byte-for-byte or bit-for-bit basis. + In practical terms, character-by-character comparisons should be done + codepoint-by-codepoint after conversion to a common character + encoding. + + False negatives are caused by the production and use of URI aliases. + Unnecessary aliases can be reduced, regardless of the comparison + method, by consistently providing URI references in an already- + normalized form (i.e., a form identical to what would be produced + after normalization is applied, as described below). + + + + +Berners-Lee, et al. Standards Track [Page 39] + +RFC 3986 URI Generic Syntax January 2005 + + + Protocols and data formats often limit some URI comparisons to simple + string comparison, based on the theory that people and + implementations will, in their own best interest, be consistent in + providing URI references, or at least consistent enough to negate any + efficiency that might be obtained from further normalization. + +6.2.2. Syntax-Based Normalization + + Implementations may use logic based on the definitions provided by + this specification to reduce the probability of false negatives. + This processing is moderately higher in cost than character-for- + character string comparison. For example, an application using this + approach could reasonably consider the following two URIs equivalent: + + example://a/b/c/%7Bfoo%7D + eXAMPLE://a/./b/../b/%63/%7bfoo%7d + + Web user agents, such as browsers, typically apply this type of URI + normalization when determining whether a cached response is + available. Syntax-based normalization includes such techniques as + case normalization, percent-encoding normalization, and removal of + dot-segments. + +6.2.2.1. Case Normalization + + For all URIs, the hexadecimal digits within a percent-encoding + triplet (e.g., "%3a" versus "%3A") are case-insensitive and therefore + should be normalized to use uppercase letters for the digits A-F. + + When a URI uses components of the generic syntax, the component + syntax equivalence rules always apply; namely, that the scheme and + host are case-insensitive and therefore should be normalized to + lowercase. For example, the URI <HTTP://www.EXAMPLE.com/> is + equivalent to <http://www.example.com/>. The other generic syntax + components are assumed to be case-sensitive unless specifically + defined otherwise by the scheme (see Section 6.2.3). + +6.2.2.2. Percent-Encoding Normalization + + The percent-encoding mechanism (Section 2.1) is a frequent source of + variance among otherwise identical URIs. In addition to the case + normalization issue noted above, some URI producers percent-encode + octets that do not require percent-encoding, resulting in URIs that + are equivalent to their non-encoded counterparts. These URIs should + be normalized by decoding any percent-encoded octet that corresponds + to an unreserved character, as described in Section 2.3. + + + + + +Berners-Lee, et al. Standards Track [Page 40] + +RFC 3986 URI Generic Syntax January 2005 + + +6.2.2.3. Path Segment Normalization + + The complete path segments "." and ".." are intended only for use + within relative references (Section 4.1) and are removed as part of + the reference resolution process (Section 5.2). However, some + deployed implementations incorrectly assume that reference resolution + is not necessary when the reference is already a URI and thus fail to + remove dot-segments when they occur in non-relative paths. URI + normalizers should remove dot-segments by applying the + remove_dot_segments algorithm to the path, as described in + Section 5.2.4. + +6.2.3. Scheme-Based Normalization + + The syntax and semantics of URIs vary from scheme to scheme, as + described by the defining specification for each scheme. + Implementations may use scheme-specific rules, at further processing + cost, to reduce the probability of false negatives. For example, + because the "http" scheme makes use of an authority component, has a + default port of "80", and defines an empty path to be equivalent to + "/", the following four URIs are equivalent: + + http://example.com + http://example.com/ + http://example.com:/ + http://example.com:80/ + + In general, a URI that uses the generic syntax for authority with an + empty path should be normalized to a path of "/". Likewise, an + explicit ":port", for which the port is empty or the default for the + scheme, is equivalent to one where the port and its ":" delimiter are + elided and thus should be removed by scheme-based normalization. For + example, the second URI above is the normal form for the "http" + scheme. + + Another case where normalization varies by scheme is in the handling + of an empty authority component or empty host subcomponent. For many + scheme specifications, an empty authority or host is considered an + error; for others, it is considered equivalent to "localhost" or the + end-user's host. When a scheme defines a default for authority and a + URI reference to that default is desired, the reference should be + normalized to an empty authority for the sake of uniformity, brevity, + and internationalization. If, however, either the userinfo or port + subcomponents are non-empty, then the host should be given explicitly + even if it matches the default. + + Normalization should not remove delimiters when their associated + component is empty unless licensed to do so by the scheme + + + +Berners-Lee, et al. Standards Track [Page 41] + +RFC 3986 URI Generic Syntax January 2005 + + + specification. For example, the URI "http://example.com/?" cannot be + assumed to be equivalent to any of the examples above. Likewise, the + presence or absence of delimiters within a userinfo subcomponent is + usually significant to its interpretation. The fragment component is + not subject to any scheme-based normalization; thus, two URIs that + differ only by the suffix "#" are considered different regardless of + the scheme. + + Some schemes define additional subcomponents that consist of case- + insensitive data, giving an implicit license to normalizers to + convert this data to a common case (e.g., all lowercase). For + example, URI schemes that define a subcomponent of path to contain an + Internet hostname, such as the "mailto" URI scheme, cause that + subcomponent to be case-insensitive and thus subject to case + normalization (e.g., "mailto:[email protected]" is equivalent to + "mailto:[email protected]", even though the generic syntax considers + the path component to be case-sensitive). + + Other scheme-specific normalizations are possible. + +6.2.4. Protocol-Based Normalization + + Substantial effort to reduce the incidence of false negatives is + often cost-effective for web spiders. Therefore, they implement even + more aggressive techniques in URI comparison. For example, if they + observe that a URI such as + + http://example.com/data + + redirects to a URI differing only in the trailing slash + + http://example.com/data/ + + they will likely regard the two as equivalent in the future. This + kind of technique is only appropriate when equivalence is clearly + indicated by both the result of accessing the resources and the + common conventions of their scheme's dereference algorithm (in this + case, use of redirection by HTTP origin servers to avoid problems + with relative references). + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 42] + +RFC 3986 URI Generic Syntax January 2005 + + +7. Security Considerations + + A URI does not in itself pose a security threat. However, as URIs + are often used to provide a compact set of instructions for access to + network resources, care must be taken to properly interpret the data + within a URI, to prevent that data from causing unintended access, + and to avoid including data that should not be revealed in plain + text. + +7.1. Reliability and Consistency + + There is no guarantee that once a URI has been used to retrieve + information, the same information will be retrievable by that URI in + the future. Nor is there any guarantee that the information + retrievable via that URI in the future will be observably similar to + that retrieved in the past. The URI syntax does not constrain how a + given scheme or authority apportions its namespace or maintains it + over time. Such guarantees can only be obtained from the person(s) + controlling that namespace and the resource in question. A specific + URI scheme may define additional semantics, such as name persistence, + if those semantics are required of all naming authorities for that + scheme. + +7.2. Malicious Construction + + It is sometimes possible to construct a URI so that an attempt to + perform a seemingly harmless, idempotent operation, such as the + retrieval of a representation, will in fact cause a possibly damaging + remote operation. The unsafe URI is typically constructed by + specifying a port number other than that reserved for the network + protocol in question. The client unwittingly contacts a site running + a different protocol service, and data within the URI contains + instructions that, when interpreted according to this other protocol, + cause an unexpected operation. A frequent example of such abuse has + been the use of a protocol-based scheme with a port component of + "25", thereby fooling user agent software into sending an unintended + or impersonating message via an SMTP server. + + Applications should prevent dereference of a URI that specifies a TCP + port number within the "well-known port" range (0 - 1023) unless the + protocol being used to dereference that URI is compatible with the + protocol expected on that well-known port. Although IANA maintains a + registry of well-known ports, applications should make such + restrictions user-configurable to avoid preventing the deployment of + new services. + + + + + + +Berners-Lee, et al. Standards Track [Page 43] + +RFC 3986 URI Generic Syntax January 2005 + + + When a URI contains percent-encoded octets that match the delimiters + for a given resolution or dereference protocol (for example, CR and + LF characters for the TELNET protocol), these percent-encodings must + not be decoded before transmission across that protocol. Transfer of + the percent-encoding, which might violate the protocol, is less + harmful than allowing decoded octets to be interpreted as additional + operations or parameters, perhaps triggering an unexpected and + possibly harmful remote operation. + +7.3. Back-End Transcoding + + When a URI is dereferenced, the data within it is often parsed by + both the user agent and one or more servers. In HTTP, for example, a + typical user agent will parse a URI into its five major components, + access the authority's server, and send it the data within the + authority, path, and query components. A typical server will take + that information, parse the path into segments and the query into + key/value pairs, and then invoke implementation-specific handlers to + respond to the request. As a result, a common security concern for + server implementations that handle a URI, either as a whole or split + into separate components, is proper interpretation of the octet data + represented by the characters and percent-encodings within that URI. + + Percent-encoded octets must be decoded at some point during the + dereference process. Applications must split the URI into its + components and subcomponents prior to decoding the octets, as + otherwise the decoded octets might be mistaken for delimiters. + Security checks of the data within a URI should be applied after + decoding the octets. Note, however, that the "%00" percent-encoding + (NUL) may require special handling and should be rejected if the + application is not expecting to receive raw data within a component. + + Special care should be taken when the URI path interpretation process + involves the use of a back-end file system or related system + functions. File systems typically assign an operational meaning to + special characters, such as the "/", "\", ":", "[", and "]" + characters, and to special device names like ".", "..", "...", "aux", + "lpt", etc. In some cases, merely testing for the existence of such + a name will cause the operating system to pause or invoke unrelated + system calls, leading to significant security concerns regarding + denial of service and unintended data transfer. It would be + impossible for this specification to list all such significant + characters and device names. Implementers should research the + reserved names and characters for the types of storage device that + may be attached to their applications and restrict the use of data + obtained from URI components accordingly. + + + + + +Berners-Lee, et al. Standards Track [Page 44] + +RFC 3986 URI Generic Syntax January 2005 + + +7.4. Rare IP Address Formats + + Although the URI syntax for IPv4address only allows the common + dotted-decimal form of IPv4 address literal, many implementations + that process URIs make use of platform-dependent system routines, + such as gethostbyname() and inet_aton(), to translate the string + literal to an actual IP address. Unfortunately, such system routines + often allow and process a much larger set of formats than those + described in Section 3.2.2. + + For example, many implementations allow dotted forms of three + numbers, wherein the last part is interpreted as a 16-bit quantity + and placed in the right-most two bytes of the network address (e.g., + a Class B network). Likewise, a dotted form of two numbers means + that the last part is interpreted as a 24-bit quantity and placed in + the right-most three bytes of the network address (Class A), and a + single number (without dots) is interpreted as a 32-bit quantity and + stored directly in the network address. Adding further to the + confusion, some implementations allow each dotted part to be + interpreted as decimal, octal, or hexadecimal, as specified in the C + language (i.e., a leading 0x or 0X implies hexadecimal; a leading 0 + implies octal; otherwise, the number is interpreted as decimal). + + These additional IP address formats are not allowed in the URI syntax + due to differences between platform implementations. However, they + can become a security concern if an application attempts to filter + access to resources based on the IP address in string literal format. + If this filtering is performed, literals should be converted to + numeric form and filtered based on the numeric value, and not on a + prefix or suffix of the string form. + +7.5. Sensitive Information + + URI producers should not provide a URI that contains a username or + password that is intended to be secret. URIs are frequently + displayed by browsers, stored in clear text bookmarks, and logged by + user agent history and intermediary applications (proxies). A + password appearing within the userinfo component is deprecated and + should be considered an error (or simply ignored) except in those + rare cases where the 'password' parameter is intended to be public. + +7.6. Semantic Attacks + + Because the userinfo subcomponent is rarely used and appears before + the host in the authority component, it can be used to construct a + URI intended to mislead a human user by appearing to identify one + (trusted) naming authority while actually identifying a different + authority hidden behind the noise. For example + + + +Berners-Lee, et al. Standards Track [Page 45] + +RFC 3986 URI Generic Syntax January 2005 + + + ftp://cnn.example.com&[email protected]/top_story.htm + + might lead a human user to assume that the host is 'cnn.example.com', + whereas it is actually '10.0.0.1'. Note that a misleading userinfo + subcomponent could be much longer than the example above. + + A misleading URI, such as that above, is an attack on the user's + preconceived notions about the meaning of a URI rather than an attack + on the software itself. User agents may be able to reduce the impact + of such attacks by distinguishing the various components of the URI + when they are rendered, such as by using a different color or tone to + render userinfo if any is present, though there is no panacea. More + information on URI-based semantic attacks can be found in [Siedzik]. + +8. IANA Considerations + + URI scheme names, as defined by <scheme> in Section 3.1, form a + registered namespace that is managed by IANA according to the + procedures defined in [BCP35]. No IANA actions are required by this + document. + +9. Acknowledgements + + This specification is derived from RFC 2396 [RFC2396], RFC 1808 + [RFC1808], and RFC 1738 [RFC1738]; the acknowledgements in those + documents still apply. It also incorporates the update (with + corrections) for IPv6 literals in the host syntax, as defined by + Robert M. Hinden, Brian E. Carpenter, and Larry Masinter in + [RFC2732]. In addition, contributions by Gisle Aas, Reese Anschultz, + Daniel Barclay, Tim Bray, Mike Brown, Rob Cameron, Jeremy Carroll, + Dan Connolly, Adam M. Costello, John Cowan, Jason Diamond, Martin + Duerst, Stefan Eissing, Clive D.W. Feather, Al Gilman, Tony Hammond, + Elliotte Harold, Pat Hayes, Henry Holtzman, Ian B. Jacobs, Michael + Kay, John C. Klensin, Graham Klyne, Dan Kohn, Bruce Lilly, Andrew + Main, Dave McAlpin, Ira McDonald, Michael Mealling, Ray Merkert, + Stephen Pollei, Julian Reschke, Tomas Rokicki, Miles Sabin, Kai + Schaetzl, Mark Thomson, Ronald Tschalaer, Norm Walsh, Marc Warne, + Stuart Williams, and Henry Zongaro are gratefully acknowledged. + +10. References + +10.1. Normative References + + [ASCII] American National Standards Institute, "Coded Character + Set -- 7-bit American Standard Code for Information + Interchange", ANSI X3.4, 1986. + + + + + +Berners-Lee, et al. Standards Track [Page 46] + +RFC 3986 URI Generic Syntax January 2005 + + + [RFC2234] Crocker, D. and P. Overell, "Augmented BNF for Syntax + Specifications: ABNF", RFC 2234, November 1997. + + [STD63] Yergeau, F., "UTF-8, a transformation format of + ISO 10646", STD 63, RFC 3629, November 2003. + + [UCS] International Organization for Standardization, + "Information Technology - Universal Multiple-Octet Coded + Character Set (UCS)", ISO/IEC 10646:2003, December 2003. + +10.2. Informative References + + [BCP19] Freed, N. and J. Postel, "IANA Charset Registration + Procedures", BCP 19, RFC 2978, October 2000. + + [BCP35] Petke, R. and I. King, "Registration Procedures for URL + Scheme Names", BCP 35, RFC 2717, November 1999. + + [RFC0952] Harrenstien, K., Stahl, M., and E. Feinler, "DoD Internet + host table specification", RFC 952, October 1985. + + [RFC1034] Mockapetris, P., "Domain names - concepts and facilities", + STD 13, RFC 1034, November 1987. + + [RFC1123] Braden, R., "Requirements for Internet Hosts - Application + and Support", STD 3, RFC 1123, October 1989. + + [RFC1535] Gavron, E., "A Security Problem and Proposed Correction + With Widely Deployed DNS Software", RFC 1535, + October 1993. + + [RFC1630] Berners-Lee, T., "Universal Resource Identifiers in WWW: A + Unifying Syntax for the Expression of Names and Addresses + of Objects on the Network as used in the World-Wide Web", + RFC 1630, June 1994. + + [RFC1736] Kunze, J., "Functional Recommendations for Internet + Resource Locators", RFC 1736, February 1995. + + [RFC1737] Sollins, K. and L. Masinter, "Functional Requirements for + Uniform Resource Names", RFC 1737, December 1994. + + [RFC1738] Berners-Lee, T., Masinter, L., and M. McCahill, "Uniform + Resource Locators (URL)", RFC 1738, December 1994. + + [RFC1808] Fielding, R., "Relative Uniform Resource Locators", + RFC 1808, June 1995. + + + + +Berners-Lee, et al. Standards Track [Page 47] + +RFC 3986 URI Generic Syntax January 2005 + + + [RFC2046] Freed, N. and N. Borenstein, "Multipurpose Internet Mail + Extensions (MIME) Part Two: Media Types", RFC 2046, + November 1996. + + [RFC2141] Moats, R., "URN Syntax", RFC 2141, May 1997. + + [RFC2396] Berners-Lee, T., Fielding, R., and L. Masinter, "Uniform + Resource Identifiers (URI): Generic Syntax", RFC 2396, + August 1998. + + [RFC2518] Goland, Y., Whitehead, E., Faizi, A., Carter, S., and D. + Jensen, "HTTP Extensions for Distributed Authoring -- + WEBDAV", RFC 2518, February 1999. + + [RFC2557] Palme, J., Hopmann, A., and N. Shelness, "MIME + Encapsulation of Aggregate Documents, such as HTML + (MHTML)", RFC 2557, March 1999. + + [RFC2718] Masinter, L., Alvestrand, H., Zigmond, D., and R. Petke, + "Guidelines for new URL Schemes", RFC 2718, November 1999. + + [RFC2732] Hinden, R., Carpenter, B., and L. Masinter, "Format for + Literal IPv6 Addresses in URL's", RFC 2732, December 1999. + + [RFC3305] Mealling, M. and R. Denenberg, "Report from the Joint + W3C/IETF URI Planning Interest Group: Uniform Resource + Identifiers (URIs), URLs, and Uniform Resource Names + (URNs): Clarifications and Recommendations", RFC 3305, + August 2002. + + [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, + "Internationalizing Domain Names in Applications (IDNA)", + RFC 3490, March 2003. + + [RFC3513] Hinden, R. and S. Deering, "Internet Protocol Version 6 + (IPv6) Addressing Architecture", RFC 3513, April 2003. + + [Siedzik] Siedzik, R., "Semantic Attacks: What's in a URL?", + April 2001, <http://www.giac.org/practical/gsec/ + Richard_Siedzik_GSEC.pdf>. + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 48] + +RFC 3986 URI Generic Syntax January 2005 + + +Appendix A. Collected ABNF for URI + + URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] + + hier-part = "//" authority path-abempty + / path-absolute + / path-rootless + / path-empty + + URI-reference = URI / relative-ref + + absolute-URI = scheme ":" hier-part [ "?" query ] + + relative-ref = relative-part [ "?" query ] [ "#" fragment ] + + relative-part = "//" authority path-abempty + / path-absolute + / path-noscheme + / path-empty + + scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + + authority = [ userinfo "@" ] host [ ":" port ] + userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + host = IP-literal / IPv4address / reg-name + port = *DIGIT + + IP-literal = "[" ( IPv6address / IPvFuture ) "]" + + IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) + + IPv6address = 6( h16 ":" ) ls32 + / "::" 5( h16 ":" ) ls32 + / [ h16 ] "::" 4( h16 ":" ) ls32 + / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + / [ *4( h16 ":" ) h16 ] "::" ls32 + / [ *5( h16 ":" ) h16 ] "::" h16 + / [ *6( h16 ":" ) h16 ] "::" + + h16 = 1*4HEXDIG + ls32 = ( h16 ":" h16 ) / IPv4address + IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + + + + + + + +Berners-Lee, et al. Standards Track [Page 49] + +RFC 3986 URI Generic Syntax January 2005 + + + dec-octet = DIGIT ; 0-9 + / %x31-39 DIGIT ; 10-99 + / "1" 2DIGIT ; 100-199 + / "2" %x30-34 DIGIT ; 200-249 + / "25" %x30-35 ; 250-255 + + reg-name = *( unreserved / pct-encoded / sub-delims ) + + path = path-abempty ; begins with "/" or is empty + / path-absolute ; begins with "/" but not "//" + / path-noscheme ; begins with a non-colon segment + / path-rootless ; begins with a segment + / path-empty ; zero characters + + path-abempty = *( "/" segment ) + path-absolute = "/" [ segment-nz *( "/" segment ) ] + path-noscheme = segment-nz-nc *( "/" segment ) + path-rootless = segment-nz *( "/" segment ) + path-empty = 0<pchar> + + segment = *pchar + segment-nz = 1*pchar + segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + ; non-zero-length segment without any colon ":" + + pchar = unreserved / pct-encoded / sub-delims / ":" / "@" + + query = *( pchar / "/" / "?" ) + + fragment = *( pchar / "/" / "?" ) + + pct-encoded = "%" HEXDIG HEXDIG + + unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + reserved = gen-delims / sub-delims + gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + / "*" / "+" / "," / ";" / "=" + +Appendix B. Parsing a URI Reference with a Regular Expression + + As the "first-match-wins" algorithm is identical to the "greedy" + disambiguation method used by POSIX regular expressions, it is + natural and commonplace to use a regular expression for parsing the + potential five components of a URI reference. + + The following line is the regular expression for breaking-down a + well-formed URI reference into its components. + + + +Berners-Lee, et al. Standards Track [Page 50] + +RFC 3986 URI Generic Syntax January 2005 + + + ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? + 12 3 4 5 6 7 8 9 + + The numbers in the second line above are only to assist readability; + they indicate the reference points for each subexpression (i.e., each + paired parenthesis). We refer to the value matched for subexpression + <n> as $<n>. For example, matching the above expression to + + http://www.ics.uci.edu/pub/ietf/uri/#Related + + results in the following subexpression matches: + + $1 = http: + $2 = http + $3 = //www.ics.uci.edu + $4 = www.ics.uci.edu + $5 = /pub/ietf/uri/ + $6 = <undefined> + $7 = <undefined> + $8 = #Related + $9 = Related + + where <undefined> indicates that the component is not present, as is + the case for the query component in the above example. Therefore, we + can determine the value of the five components as + + scheme = $2 + authority = $4 + path = $5 + query = $7 + fragment = $9 + + Going in the opposite direction, we can recreate a URI reference from + its components by using the algorithm of Section 5.3. + +Appendix C. Delimiting a URI in Context + + URIs are often transmitted through formats that do not provide a + clear context for their interpretation. For example, there are many + occasions when a URI is included in plain text; examples include text + sent in email, USENET news, and on printed paper. In such cases, it + is important to be able to delimit the URI from the rest of the text, + and in particular from punctuation marks that might be mistaken for + part of the URI. + + In practice, URIs are delimited in a variety of ways, but usually + within double-quotes "http://example.com/", angle brackets + <http://example.com/>, or just by using whitespace: + + + +Berners-Lee, et al. Standards Track [Page 51] + +RFC 3986 URI Generic Syntax January 2005 + + + http://example.com/ + + These wrappers do not form part of the URI. + + In some cases, extra whitespace (spaces, line-breaks, tabs, etc.) may + have to be added to break a long URI across lines. The whitespace + should be ignored when the URI is extracted. + + No whitespace should be introduced after a hyphen ("-") character. + Because some typesetters and printers may (erroneously) introduce a + hyphen at the end of line when breaking it, the interpreter of a URI + containing a line break immediately after a hyphen should ignore all + whitespace around the line break and should be aware that the hyphen + may or may not actually be part of the URI. + + Using <> angle brackets around each URI is especially recommended as + a delimiting style for a reference that contains embedded whitespace. + + The prefix "URL:" (with or without a trailing space) was formerly + recommended as a way to help distinguish a URI from other bracketed + designators, though it is not commonly used in practice and is no + longer recommended. + + For robustness, software that accepts user-typed URI should attempt + to recognize and strip both delimiters and embedded whitespace. + + For example, the text + + Yes, Jim, I found it under "http://www.w3.org/Addressing/", + but you can probably pick it up from <ftp://foo.example. + com/rfc/>. Note the warning in <http://www.ics.uci.edu/pub/ + ietf/uri/historical.html#WARNING>. + + contains the URI references + + http://www.w3.org/Addressing/ + ftp://foo.example.com/rfc/ + http://www.ics.uci.edu/pub/ietf/uri/historical.html#WARNING + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 52] + +RFC 3986 URI Generic Syntax January 2005 + + +Appendix D. Changes from RFC 2396 + +D.1. Additions + + An ABNF rule for URI has been introduced to correspond to one common + usage of the term: an absolute URI with optional fragment. + + IPv6 (and later) literals have been added to the list of possible + identifiers for the host portion of an authority component, as + described by [RFC2732], with the addition of "[" and "]" to the + reserved set and a version flag to anticipate future versions of IP + literals. Square brackets are now specified as reserved within the + authority component and are not allowed outside their use as + delimiters for an IP literal within host. In order to make this + change without changing the technical definition of the path, query, + and fragment components, those rules were redefined to directly + specify the characters allowed. + + As [RFC2732] defers to [RFC3513] for definition of an IPv6 literal + address, which, unfortunately, lacks an ABNF description of + IPv6address, we created a new ABNF rule for IPv6address that matches + the text representations defined by Section 2.2 of [RFC3513]. + Likewise, the definition of IPv4address has been improved in order to + limit each decimal octet to the range 0-255. + + Section 6, on URI normalization and comparison, has been completely + rewritten and extended by using input from Tim Bray and discussion + within the W3C Technical Architecture Group. + +D.2. Modifications + + The ad-hoc BNF syntax of RFC 2396 has been replaced with the ABNF of + [RFC2234]. This change required all rule names that formerly + included underscore characters to be renamed with a dash instead. In + addition, a number of syntax rules have been eliminated or simplified + to make the overall grammar more comprehensible. Specifications that + refer to the obsolete grammar rules may be understood by replacing + those rules according to the following table: + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 53] + +RFC 3986 URI Generic Syntax January 2005 + + + +----------------+--------------------------------------------------+ + | obsolete rule | translation | + +----------------+--------------------------------------------------+ + | absoluteURI | absolute-URI | + | relativeURI | relative-part [ "?" query ] | + | hier_part | ( "//" authority path-abempty / | + | | path-absolute ) [ "?" query ] | + | | | + | opaque_part | path-rootless [ "?" query ] | + | net_path | "//" authority path-abempty | + | abs_path | path-absolute | + | rel_path | path-rootless | + | rel_segment | segment-nz-nc | + | reg_name | reg-name | + | server | authority | + | hostport | host [ ":" port ] | + | hostname | reg-name | + | path_segments | path-abempty | + | param | *<pchar excluding ";"> | + | | | + | uric | unreserved / pct-encoded / ";" / "?" / ":" | + | | / "@" / "&" / "=" / "+" / "$" / "," / "/" | + | | | + | uric_no_slash | unreserved / pct-encoded / ";" / "?" / ":" | + | | / "@" / "&" / "=" / "+" / "$" / "," | + | | | + | mark | "-" / "_" / "." / "!" / "~" / "*" / "'" | + | | / "(" / ")" | + | | | + | escaped | pct-encoded | + | hex | HEXDIG | + | alphanum | ALPHA / DIGIT | + +----------------+--------------------------------------------------+ + + Use of the above obsolete rules for the definition of scheme-specific + syntax is deprecated. + + Section 2, on characters, has been rewritten to explain what + characters are reserved, when they are reserved, and why they are + reserved, even when they are not used as delimiters by the generic + syntax. The mark characters that are typically unsafe to decode, + including the exclamation mark ("!"), asterisk ("*"), single-quote + ("'"), and open and close parentheses ("(" and ")"), have been moved + to the reserved set in order to clarify the distinction between + reserved and unreserved and, hopefully, to answer the most common + question of scheme designers. Likewise, the section on + percent-encoded characters has been rewritten, and URI normalizers + are now given license to decode any percent-encoded octets + + + +Berners-Lee, et al. Standards Track [Page 54] + +RFC 3986 URI Generic Syntax January 2005 + + + corresponding to unreserved characters. In general, the terms + "escaped" and "unescaped" have been replaced with "percent-encoded" + and "decoded", respectively, to reduce confusion with other forms of + escape mechanisms. + + The ABNF for URI and URI-reference has been redesigned to make them + more friendly to LALR parsers and to reduce complexity. As a result, + the layout form of syntax description has been removed, along with + the uric, uric_no_slash, opaque_part, net_path, abs_path, rel_path, + path_segments, rel_segment, and mark rules. All references to + "opaque" URIs have been replaced with a better description of how the + path component may be opaque to hierarchy. The relativeURI rule has + been replaced with relative-ref to avoid unnecessary confusion over + whether they are a subset of URI. The ambiguity regarding the + parsing of URI-reference as a URI or a relative-ref with a colon in + the first segment has been eliminated through the use of five + separate path matching rules. + + The fragment identifier has been moved back into the section on + generic syntax components and within the URI and relative-ref rules, + though it remains excluded from absolute-URI. The number sign ("#") + character has been moved back to the reserved set as a result of + reintegrating the fragment syntax. + + The ABNF has been corrected to allow the path component to be empty. + This also allows an absolute-URI to consist of nothing after the + "scheme:", as is present in practice with the "dav:" namespace + [RFC2518] and with the "about:" scheme used internally by many WWW + browser implementations. The ambiguity regarding the boundary + between authority and path has been eliminated through the use of + five separate path matching rules. + + Registry-based naming authorities that use the generic syntax are now + defined within the host rule. This change allows current + implementations, where whatever name provided is simply fed to the + local name resolution mechanism, to be consistent with the + specification. It also removes the need to re-specify DNS name + formats here. Furthermore, it allows the host component to contain + percent-encoded octets, which is necessary to enable + internationalized domain names to be provided in URIs, processed in + their native character encodings at the application layers above URI + processing, and passed to an IDNA library as a registered name in the + UTF-8 character encoding. The server, hostport, hostname, + domainlabel, toplabel, and alphanum rules have been removed. + + The resolving relative references algorithm of [RFC2396] has been + rewritten with pseudocode for this revision to improve clarity and + fix the following issues: + + + +Berners-Lee, et al. Standards Track [Page 55] + +RFC 3986 URI Generic Syntax January 2005 + + + o [RFC2396] section 5.2, step 6a, failed to account for a base URI + with no path. + + o Restored the behavior of [RFC1808] where, if the reference + contains an empty path and a defined query component, the target + URI inherits the base URI's path component. + + o The determination of whether a URI reference is a same-document + reference has been decoupled from the URI parser, simplifying the + URI processing interface within applications in a way consistent + with the internal architecture of deployed URI processing + implementations. The determination is now based on comparison to + the base URI after transforming a reference to absolute form, + rather than on the format of the reference itself. This change + may result in more references being considered "same-document" + under this specification than there would be under the rules given + in RFC 2396, especially when normalization is used to reduce + aliases. However, it does not change the status of existing + same-document references. + + o Separated the path merge routine into two routines: merge, for + describing combination of the base URI path with a relative-path + reference, and remove_dot_segments, for describing how to remove + the special "." and ".." segments from a composed path. The + remove_dot_segments algorithm is now applied to all URI reference + paths in order to match common implementations and to improve the + normalization of URIs in practice. This change only impacts the + parsing of abnormal references and same-scheme references wherein + the base URI has a non-hierarchical path. + +Index + + A + ABNF 11 + absolute 27 + absolute-path 26 + absolute-URI 27 + access 9 + authority 17, 18 + + B + base URI 28 + + C + character encoding 4 + character 4 + characters 8, 11 + coded character set 4 + + + +Berners-Lee, et al. Standards Track [Page 56] + +RFC 3986 URI Generic Syntax January 2005 + + + D + dec-octet 20 + dereference 9 + dot-segments 23 + + F + fragment 16, 24 + + G + gen-delims 13 + generic syntax 6 + + H + h16 20 + hier-part 16 + hierarchical 10 + host 18 + + I + identifier 5 + IP-literal 19 + IPv4 20 + IPv4address 19, 20 + IPv6 19 + IPv6address 19, 20 + IPvFuture 19 + + L + locator 7 + ls32 20 + + M + merge 32 + + N + name 7 + network-path 26 + + P + path 16, 22, 26 + path-abempty 22 + path-absolute 22 + path-empty 22 + path-noscheme 22 + path-rootless 22 + path-abempty 16, 22, 26 + path-absolute 16, 22, 26 + path-empty 16, 22, 26 + + + +Berners-Lee, et al. Standards Track [Page 57] + +RFC 3986 URI Generic Syntax January 2005 + + + path-rootless 16, 22 + pchar 23 + pct-encoded 12 + percent-encoding 12 + port 22 + + Q + query 16, 23 + + R + reg-name 21 + registered name 20 + relative 10, 28 + relative-path 26 + relative-ref 26 + remove_dot_segments 33 + representation 9 + reserved 12 + resolution 9, 28 + resource 5 + retrieval 9 + + S + same-document 27 + sameness 9 + scheme 16, 17 + segment 22, 23 + segment-nz 23 + segment-nz-nc 23 + sub-delims 13 + suffix 27 + + T + transcription 8 + + U + uniform 4 + unreserved 13 + URI grammar + absolute-URI 27 + ALPHA 11 + authority 18 + CR 11 + dec-octet 20 + DIGIT 11 + DQUOTE 11 + fragment 24 + gen-delims 13 + + + +Berners-Lee, et al. Standards Track [Page 58] + +RFC 3986 URI Generic Syntax January 2005 + + + h16 20 + HEXDIG 11 + hier-part 16 + host 19 + IP-literal 19 + IPv4address 20 + IPv6address 20 + IPvFuture 19 + LF 11 + ls32 20 + OCTET 11 + path 22 + path-abempty 22 + path-absolute 22 + path-empty 22 + path-noscheme 22 + path-rootless 22 + pchar 23 + pct-encoded 12 + port 22 + query 24 + reg-name 21 + relative-ref 26 + reserved 13 + scheme 17 + segment 23 + segment-nz 23 + segment-nz-nc 23 + SP 11 + sub-delims 13 + unreserved 13 + URI 16 + URI-reference 25 + userinfo 18 + URI 16 + URI-reference 25 + URL 7 + URN 7 + userinfo 18 + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 59] + +RFC 3986 URI Generic Syntax January 2005 + + +Authors' Addresses + + Tim Berners-Lee + World Wide Web Consortium + Massachusetts Institute of Technology + 77 Massachusetts Avenue + Cambridge, MA 02139 + USA + + Phone: +1-617-253-5702 + Fax: +1-617-258-5999 + EMail: [email protected] + URI: http://www.w3.org/People/Berners-Lee/ + + + Roy T. Fielding + Day Software + 5251 California Ave., Suite 110 + Irvine, CA 92617 + USA + + Phone: +1-949-679-2960 + Fax: +1-949-679-2972 + EMail: [email protected] + URI: http://roy.gbiv.com/ + + + Larry Masinter + Adobe Systems Incorporated + 345 Park Ave + San Jose, CA 95110 + USA + + Phone: +1-408-536-3024 + EMail: [email protected] + URI: http://larry.masinter.net/ + + + + + + + + + + + + + + + +Berners-Lee, et al. Standards Track [Page 60] + +RFC 3986 URI Generic Syntax January 2005 + + +Full Copyright Statement + + Copyright (C) The Internet Society (2005). + + This document is subject to the rights, licenses and restrictions + contained in BCP 78, and except as set forth therein, the authors + retain all their rights. + + This document and the information contained herein are provided on an + "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS + OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET + ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE + INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED + WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Intellectual Property + + The IETF takes no position regarding the validity or scope of any + Intellectual Property Rights or other rights that might be claimed to + pertain to the implementation or use of the technology described in + this document or the extent to which any license under such rights + might or might not be available; nor does it represent that it has + made any independent effort to identify any such rights. Information + on the IETF's procedures with respect to rights in IETF Documents can + be found in BCP 78 and BCP 79. + + Copies of IPR disclosures made to the IETF Secretariat and any + assurances of licenses to be made available, or the result of an + attempt made to obtain a general license or permission for the use of + such proprietary rights by implementers or users of this + specification can be obtained from the IETF on-line IPR repository at + http://www.ietf.org/ipr. + + The IETF invites any interested party to bring to its attention any + copyrights, patents or patent applications, or other proprietary + rights that may cover technology that may be required to implement + this standard. Please address the information to the IETF at ietf- + + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + +Berners-Lee, et al. Standards Track [Page 61] + diff --git a/lib/inets/doc/src/Makefile b/lib/inets/doc/src/Makefile index e4cb0c4e48..82f2a5829f 100644 --- a/lib/inets/doc/src/Makefile +++ b/lib/inets/doc/src/Makefile @@ -26,16 +26,6 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk include ../../vsn.mk VSN=$(INETS_VSN) - -# ---------------------------------------------------- -# Include dependency -# ---------------------------------------------------- - -ifndef DOCSUPPORT -include make.dep -endif - - # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- @@ -98,37 +88,10 @@ EXTRA_FILES = summary.html.src \ MAN3_FILES = $(XML_REF3_FILES:%.xml=$(MAN3DIR)/%.3) -ifdef DOCSUPPORT - HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf -else - -TEX_FILES_BOOK = \ - $(BOOK_FILES:%.xml=%.tex) -TEX_FILES_REF_MAN = \ - $(XML_PART_FILES:%.xml=%.tex) \ - $(XML_REF3_FILES:%.xml=%.tex) \ - $(XML_REF6_FILES:%.xml=%.tex) \ - $(XML_APPLICATION_FILES:%.xml=%.tex) -TEX_FILES_USERS_GUIDE = \ - $(XML_CHAPTER_FILES:%.xml=%.tex) - -TOP_PDF_FILE = $(APPLICATION)-$(VSN).pdf -TOP_PS_FILE = $(APPLICATION)-$(VSN).ps - -$(TOP_PDF_FILE): book.dvi ../../vsn.mk - $(DVI2PS) $(DVIPS_FLAGS) -f $< | $(DISTILL) $(DISTILL_FLAGS) > $@ - -$(TOP_PS_FILE): book.dvi ../../vsn.mk - $(DVI2PS) $(DVIPS_FLAGS) -f $< > $@ - -TOP_HTML_FILES = - -endif - # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- @@ -141,8 +104,6 @@ DVIPS_FLAGS += $(HTMLDIR)/%.gif: %.gif $(INSTALL_DATA) $< $@ -ifdef DOCSUPPORT - docs: pdf html man ldocs: local_docs @@ -156,33 +117,6 @@ html: gifs $(HTML_REF_MAN_FILE) clean clean_docs: clean_html clean_man clean_pdf rm -f errs core *~ -else - -ifeq ($(DOCTYPE),pdf) -docs: pdf -else -ifeq ($(DOCTYPE),ps) -docs: ps -else -docs: html man -endif -endif - -pdf: $(TOP_PDF_FILE) - -ps: $(TOP_PS_FILE) - -html: $(HTML_FILES) $(TOP_HTML_FILES) gifs - -clean_tex: - rm -f $(TEX_FILES_USERS_GUIDE) $(TEX_FILES_REF_MAN) $(TEX_FILES_BOOK) - -clean: clean_tex clean_html clean_man - rm -f *.xmls_output *.xmls_errs - rm -f $(TOP_PDF_FILE) - rm -f errs core *~ -endif - man: $(MAN3_FILES) gifs: $(GIF_FILES:%=$(HTMLDIR)/%) @@ -204,10 +138,7 @@ clean_man: # ---------------------------------------------------- include $(ERL_TOP)/make/otp_release_targets.mk -ifdef DOCSUPPORT - release_docs_spec: docs - @echo "release_docs_spec(docs) when DOCSUPPORT=$DOCSUPPORT" $(INSTALL_DIR) $(RELSYSDIR)/doc/pdf $(INSTALL_DATA) $(TOP_PDF_FILE) $(RELSYSDIR)/doc/pdf $(INSTALL_DIR) $(RELSYSDIR)/doc/html @@ -215,33 +146,6 @@ release_docs_spec: docs $(INSTALL_DATA) $(INFO_FILE) $(RELSYSDIR) $(INSTALL_DIR) $(RELEASE_PATH)/man/man3 $(INSTALL_DATA) $(MAN3DIR)/* $(RELEASE_PATH)/man/man3 -else - -ifeq ($(DOCTYPE),pdf) -release_docs_spec: pdf - @echo "release_docs_spec(pdf)" - $(INSTALL_DIR) $(RELEASE_PATH)/pdf - $(INSTALL_DATA) $(TOP_PDF_FILE) $(RELEASE_PATH)/pdf -else -ifeq ($(DOCTYPE),ps) -release_docs_spec: ps - @echo "release_docs_spec(ps)" - $(INSTALL_DIR) $(RELEASE_PATH)/ps - $(INSTALL_DATA) $(TOP_PS_FILE) $(RELEASE_PATH)/ps -else -release_docs_spec: docs - @echo "release_docs_spec(docs)" - $(INSTALL_DIR) $(RELSYSDIR)/doc/html - $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTML_FILES) \ - $(RELSYSDIR)/doc/html - $(INSTALL_DATA) $(INFO_FILE) $(RELSYSDIR) - $(INSTALL_DIR) $(RELEASE_PATH)/man/man3 - $(INSTALL_DATA) $(MAN3_FILES) $(RELEASE_PATH)/man/man3 - -endif -endif - -endif release_spec: diff --git a/lib/inets/doc/src/ftp.xml b/lib/inets/doc/src/ftp.xml index ca902d8d9d..f8f11ec705 100644 --- a/lib/inets/doc/src/ftp.xml +++ b/lib/inets/doc/src/ftp.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="iso-8859-1" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> <header> <copyright> - <year>1997</year><year>2010</year> + <year>1997</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -141,11 +141,21 @@ <tag>{timeout, Timeout}</tag> <item> <marker id="timeout"></marker> - <p>Timeout = <c>integer() >= 0</c> </p> + <p>Timeout = <c>non_neg_integer()</c> </p> <p>Connection timeout. </p> <p>Default is 60000 (milliseconds). </p> </item> + <tag>{dtimeout, DTimeout}</tag> + <item> + <marker id="dtimeout"></marker> + <p>DTimeout = <c>non_neg_integer() | infinity</c> </p> + <p>Data Connect timeout. + The time the client will wait for the server to connect to the + data socket. </p> + <p>Default is infinity. </p> + </item> + <tag>{progress, Progress}</tag> <item> <marker id="progress"></marker> @@ -542,11 +552,12 @@ <v>verbose() = boolean() (defaults to false)</v> <v>debug() = disable | debug | trace (defaults to disable)</v> <!-- <v>open_options() = [open_option()]</v> --> - <v>open_option() = {ipfamily, ipfamily()} | {port, port()} | {mode, mode()} | {timeout, timeout()} | {progress, progress()}</v> + <v>open_option() = {ipfamily, ipfamily()} | {port, port()} | {mode, mode()} | {timeout, timeout()} | {dtimeout, dtimeout()} | {progress, progress()}</v> <v>ipfamily() = inet | inet6 | inet6fb4 (defaults to inet)</v> <v>port() = integer() > 0 (defaults to 21)</v> <v>mode() = active | passive (defaults to passive)</v> - <v>timeout() = integer() >= 0 (defaults to 60000 milliseconds)</v> + <v>timeout() = integer() > 0 (defaults to 60000 milliseconds)</v> + <v>dtimeout() = integer() > 0 | infinity (defaults to infinity)</v> <v>pogress() = ignore | {module(), function(), initial_data()} (defaults to ignore)</v> <v>module() = atom()</v> <v>function() = atom()</v> diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index d1671ac9bd..b1f964ae69 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -28,8 +28,10 @@ <date></date> <rev></rev> </header> + <module>httpc</module> <modulesummary>An HTTP/1.1 client </modulesummary> + <description> <p>This module provides the API to a HTTP/1.1 compatible client according to RFC 2616, caching is currently not supported.</p> @@ -167,7 +169,6 @@ filename() = string() <v>http_option() = {timeout, timeout()} | {connect_timeout, timeout()} | {ssl, ssloptions()} | - {ossl, ssloptions()} | {essl, ssloptions()} | {autoredirect, boolean()} | {proxy_auth, {userstring(), passwordstring()}} | @@ -206,6 +207,7 @@ filename() = string() to the <c>receiver</c> depending on that value. </p> <p>Http option (<c>http_option()</c>) details: </p> + <marker id="request2_http_options"></marker> <taglist> <tag><c><![CDATA[timeout]]></c></tag> <item> @@ -231,16 +233,9 @@ filename() = string() <p>Defaults to <c>[]</c>. </p> </item> - <tag><c><![CDATA[ossl]]></c></tag> - <item> - <p>If using the OpenSSL based (old) implementation of SSL, - these SSL-specific options are used. </p> - <p>Defaults to <c>[]</c>. </p> - </item> - <tag><c><![CDATA[essl]]></c></tag> <item> - <p>If using the Erlang based (new) implementation of SSL, + <p>If using the Erlang based implementation of SSL, these SSL-specific options are used. </p> <p>Defaults to <c>[]</c>. </p> </item> diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index edacb73b65..f88099a82e 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -148,13 +148,11 @@ in the apache like configuration file. </item> - <tag>{socket_type, ip_comm | ssl | ossl | essl}</tag> + <tag>{socket_type, ip_comm | ssl | essl}</tag> <item> - <p>When using ssl, there are several alternatives. - <c>ossl</c> specifically uses the OpenSSL based (old) SSL. - <c>essl</c> specifically uses the Erlang based (new) SSL. - When using <c>ssl</c> it <em>currently</em> defaults to - <c>essl</c>. </p> + <p>When using ssl, there are currently only one alternative. + <c>essl</c> specifically uses the Erlang based SSL. + <c>ssl</c> defaults to <c>essl</c>. </p> <p>Defaults to <c>ip_comm</c>. </p> </item> @@ -162,7 +160,7 @@ <item> <p>Defaults to <c>inet6fb4. </c> </p> <p>Note that this option is only used when the option - <c>socket_type</c> has the value <c>ip_comm</c>. </p> + <c>socket_type</c> has the value <c>ip_comm</c>. </p> </item> </taglist> diff --git a/lib/inets/doc/src/make.dep b/lib/inets/doc/src/make.dep deleted file mode 100644 index 8deb7e7a5a..0000000000 --- a/lib/inets/doc/src/make.dep +++ /dev/null @@ -1,47 +0,0 @@ -# -# %CopyrightBegin% -# -# Copyright Ericsson AB 1999-2010. All Rights Reserved. -# -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. -# -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. -# -# %CopyrightEnd% -# -# - -# ---------------------------------------------------- -# >>>> Do not edit this file <<<< -# This file was automaticly generated by -# /home/otp/bin/docdepend -# ---------------------------------------------------- - - -# ---------------------------------------------------- -# TeX files that the DVI file depend on -# ---------------------------------------------------- - -book.dvi: book.tex ftp.tex ftp_client.tex httpc.tex http_client.tex \ - http_server.tex httpd.tex httpd_conf.tex httpd_socket.tex \ - httpd_util.tex inets.tex inets_services.tex \ - mod_alias.tex mod_auth.tex mod_esi.tex mod_security.tex \ - part.tex ref_man.tex tftp.tex - -# ---------------------------------------------------- -# Source inlined when transforming from source to LaTeX -# ---------------------------------------------------- - -book.tex: ref_man.xml - -ftp.tex: ../../../../system/doc/definitions/term.defs - -inets_services.tex: ../../../../system/doc/definitions/term.defs - diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 34f26bf45b..7f0f61148c 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -32,6 +32,189 @@ <file>notes.xml</file> </header> + <section><title>Inets 5.8</title> + + <section><title>Improvements and New Features</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[ftpc] Add a config option to specify a + <seealso marker="ftp#dtimeout">data connect timeout</seealso>. + That is how long the ftp client will wait for the server to connect + to the data socket. If this timeout occurs, an error will be + returned to the caller and the ftp client process will be + terminated. </p> + <p>Own Id: OTP-9545</p> + </item> + + </list> + + </section> + + <section><title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[httpd] Fix logging of content length in mod_log. </p> + <p>Garrett Smith</p> + <p>Own Id: OTP-9715</p> + </item> + + </list> + + </section> + + <section> + <title>Incompatibilities</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[httpc] Deprecated interface module <c>http</c> has been removed. + It has (long) been replaced by http client interface module + <seealso marker="httpc#">httpc</seealso>. </p> + <p>Own Id: OTP-9359</p> + </item> + + <item> + <p>[httpc|httpd] The old ssl implementation (based on OpenSSL), + has been deprecated. The config option that specified usage of + this version of the ssl app, <c>ossl</c>, has been removed. </p> + <p>Own Id: OTP-9522</p> + </item> + + </list> + + </section> + + </section> <!-- 5.8 --> + + + <section><title>Inets 5.7.2</title> + <section><title>Improvements and New Features</title> + <p>-</p> + +<!-- + <list> + <item> + <p>[httpc|httpd] Added support for IPv6 with ssl. </p> + <p>Own Id: OTP-5566</p> + </item> + + </list> +--> + + </section> + + <section> + <title>Incompatibilities</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[httpc] Deprecated interface module <c>http</c> has been removed. + It has (long) been replaced by http client interface module + <seealso marker="httpc#">httpc</seealso>. </p> + <p>Own Id: OTP-9359</p> + </item> + + <item> + <p>[httpc|httpd] The old ssl implementation (based on OpenSSL), + has been deprecated. The config option that specified usage of + this version of the ssl app, <c>ossl</c>, has been removed. </p> + <p>Own Id: OTP-9522</p> + </item> + + </list> + + </section> + + <section><title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[httpd] XSS prevention did not work for hex-encoded URL's. </p> + <p>Own Id: OTP-9655</p> + </item> + + <item> + <p>[httpd] GET request with malformed header date caused + server crash (non-fatal) with no reply to client. Will + now result in a reply with status code 400. </p> + <p>Own Id: OTP-9674</p> + <p>Aux Id: seq11936</p> + </item> + + </list> + </section> + + </section> <!-- 5.7.2 --> + + + <section><title>Inets 5.7.1</title> + + <section><title>Improvements and New Features</title> + <p>-</p> + +<!-- + <list> + <item> + <p>[httpc|httpd] Added support for IPv6 with ssl. </p> + <p>Own Id: OTP-5566</p> + </item> + + </list> +--> + + </section> + + <section><title>Fixed Bugs and Malfunctions</title> +<!-- + <p>-</p> +--> + + <list> + <item> + <p>[httpc] Parsing of a cookie expire date should be more forgiving. + That is, if the parsing fails, the date should be ignored. + Also added support for (yet another) date format: + "Tue Jan 01 08:00:01 2036 GMT". </p> + <p>Own Id: OTP-9433</p> + </item> + + <item> + <p>[httpc] Rewrote cookie parsing. Among other things solving + cookie processing from www.expedia.com. </p> + <p>Own Id: OTP-9434</p> + </item> + + <item> + <p>[httpd] Fix httpd directory traversal on Windows. + Directory traversal was possible on Windows where + backward slash is used as directory separator. </p> + <p>Andr�s Veres-Szentkir�lyi.</p> + <p>Own Id: OTP-9561</p> + </item> + + </list> + </section> + + </section> <!-- 5.7.1 --> + + <section><title>Inets 5.7</title> <section><title>Improvements and New Features</title> @@ -1044,570 +1227,11 @@ </section> <!-- 5.1 --> + <!-- + <p>For information about older versions see + <url href="part_notes_history_frame.html">release notes history</url>.</p> + --> - <section><title>Inets 5.0.14</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [tftp] The callback watchdog has been removed, as it - turned out to be counter productive when the disk was - overloaded. Earlier a connection was aborted when a - callback (which performs the file access in the TFTP - server) took too long time.</p> - <p> - [tftp] The error message "Too many connections" has been - reclassified to be a warning.</p> - <p> - Own Id: OTP-7888</p> - </item> - </list> - </section> - - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p>[httpc] - Incorrect http version option check. </p> - <p>Mats Cronqvist</p> - <p>Own Id: OTP-7882</p> - </item> - - <item> - <p>[httpc] - Unnecessary error report when client - terminating as a result of the server closed the - socket unexpectedly. </p> - <p>Own Id: OTP-7883</p> - </item> - - <item> - <p>[httpc] - Failed transforming a relative URI to - an absolute URI. </p> - <p>[email protected]</p> - <p>Own Id: OTP-7950</p> - </item> - - <item> - <p>[httpd] - The HTTP server did not handle the config - option ssl_ca_certificate_file. </p> - <p>[email protected]</p> - <p>Own Id: OTP-7976</p> - </item> - - </list> - </section> - - </section> <!-- 5.0.14 --> - - - <section><title>Inets 5.0.13</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Ssl did not work correctly with the use of new style - configuration due to sn old internal format that was not - changed correctly in all places.</p> - <p> - Own Id: OTP-7723 Aux Id: seq11143 </p> - </item> - <item> - <p> - [httpc] - Now streams 200 and 206 results and not only - 200 results.</p> - <p> - Own Id: OTP-7857</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [httpc] - The inets http client will now use persistent - connections without pipelining as default and if a - pipeline timeout is set it will pipeline the requests on - the persistent connections.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-7463</p> - </item> - <item> - <p> - [httpd] - added option ssl_password_callback_arguments.</p> - <p> - Own Id: OTP-7724 Aux Id: seq11151 </p> - </item> - <item> - <p> - Changed the socket use so that it will become more robust - to non-functional ipv6 and fallback on ipv4. This changes - may for very special os-configurations cause a problem - when used with erts-versions pre R13.</p> - <p> - Own Id: OTP-7726</p> - </item> - <item> - <p> - Removed deprecated function httpd_util:key1search/[2,3]</p> - <p> - Own Id: OTP-7815</p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0.12</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [httpd] - Updated inets so that it not uses the deprecated - function ssl:accept/[2,3].</p> - <p> - Own Id: OTP-7636 Aux Id: seq11086 </p> - </item> - </list> - </section> - - </section> - - - <section><title>Inets 5.0.11</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Transient bug related to hot code swap of the TFTP server is - now fixed. It could happen that the first TFTP server that was - started after a code upgrade to Inets-5.0.6 crashed with a - function clause error in tftp_engine:service_init/2.</p> - <p> Own Id: OTP-7574 Aux Id: seq11069 </p> - </item> - <item> - <p> - [httpd] - Validation of ssl_password_callback_module was - incorrect.</p> - <p> - Own Id: OTP-7597 Aux Id: seq11074 </p> - </item> - <item> - <p> - [httpd] - Misspelling in old apachelike configuration - directive TransferDiskLogSize has been corrected.</p> - <p> Own Id: OTP-7598 Aux Id: seq11059 </p> - </item> - <item> - <p> - Minor problems found by dialyzer has been fixed.</p> - <p> - Own Id: OTP-7605</p> - </item> - </list> - </section> - - </section> - -<section><title>Inets 5.0.10</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Enhanched an info report.</p> - <p> - Own Id: OTP-7450</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Changed errro message from - {wrong_type,{document_root,"/tmp/htdocs"}} to - {invalid_option,{non_existing, - document_root,"/tmp/htdocs"}}.</p> - <p> - Own Id: OTP-7454</p> - </item> - <item> - <p> - Relative paths in directory authentication did not work - as intended, this has now been fixed.</p> - <p> - Own Id: OTP-7490</p> - </item> - <item> - <p> - The query-string passed to the callback function was not - compliant with the documentation, it is now.</p> - <p> - Own Id: OTP-7512</p> - </item> - </list> - </section> - -</section> - - <section><title>Inets 5.0.9</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Parameters to error_logger:error_report/1 has been - corrected.</p> - <p> - Own Id: OTP-7257 Aux Id: OTP-7294, OTP-7258 </p> - </item> - <item> - <p> - [httpd] - If a Module/Function request matching an - erl_script_alias registration does not exist as a function in - the module registered a 404 error will now be issued instead of a - 500 error.</p> - <p> - Own Id: OTP-7323</p> - </item> - <item> - <p> - [httpd] -The option auth_type for mod_auth is no longer - mandatory, for backward-compatibility reasons.</p> - <p> - Own Id: OTP-7341</p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0.8</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - [httpd] - Spelling error caused client connection header - to be ignored.</p> - <p> - Own Id: OTP-7315 Aux Id: seq10951 </p> - </item> - <item> - <p> - [httpd] - Call to the function - mod_get:get_modification_date/1 was made too early - resulting in that httpd did not send the 404 file missing - response.</p> - <p> - Own Id: OTP-7321</p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0.7</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [httpc, httpd] - Now follows the recommendation regarding - line terminators in section 19.3 in RFC 2616 e.i: "The - line terminator for message-header fields is the sequence - CRLF. However, we recommend that applications, when - parsing such headers, recognize a single LF as a line - terminator and ignore the leading CR".</p> - <p> - Own Id: OTP-7304 Aux Id: seq10944 </p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0.6</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [tftp] If a callback (which performs the file access in - the TFTP server) takes too long time (more than the - double TFTP timeout), the server will abort the - connection and send an error reply to the client. This - implies that the server will release resources attached - to the connection faster than before. The server simply - assumes that the client has given up.</p> - <p> - [tftp] If the TFTP server receives yet another request - from the same client (same host and port) while it - already has an active connection to the client, it will - simply ignore the new request if the request is equal - with the first one (same filename and options). This - implies that the (new) client will be served by the - already ongoing connection on the server side. By not - setting up yet another connection, in parallel with the - ongoing one, the server will consumer lesser resources.</p> - <p> - [tftp] netascii mode is now supported when the - client/server has native ascii support (Windows). The new - optional parameter native_ascii in the tftp_binary and - tftp_file callback modules can be used to override the - default behavior.</p> - <p> - [tftp] Yet another callback module has been added in - order to allow customized handling of error, warning and - info messages. See the new configuration parameter, - logger.</p> - <p> - [tftp] Yet another configuration parameter, max_retries, - has been added in order to control the number of times a - packet can be resent. The default is 5.</p> - <p> - [tftp] tftp:info/1 and tftp:change_config/2 can now be - applied to all daemons or all servers in one command - without bothering about their process identifiers.</p> - <p> - External TR HI89527.</p> - <p> - Own Id: OTP-7266</p> - </item> - </list> - </section> - -</section> - -<section><title>Inets 5.0.5</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [tftp] Blocks with too low block numbers are silently - discarded. For example if a server receives block #5 when - it expects block #7 it will discard the block without - interrupting the file transfer. Too high block numbers - does still imply an error. External TR HI96072.</p> - <p> - Own Id: OTP-7220</p> - </item> - <item> - <p> - [tftp] The problem with occasional case_clause errors in - tftp_engine:common_read/7 has been fixed. External TR - HI97362.</p> - <p> - Own Id: OTP-7221</p> - </item> - </list> - </section> - -</section> - - <section><title>Inets 5.0.4</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Changed calls to file open to concur with the API and not - use deprecated syntax.</p> - <p> - Own Id: OTP-7172</p> - </item> - <item> - <p> - [tftp] Server lost the first packet when the client timed - out</p> - <p> - Own Id: OTP-7173</p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0.3</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Updated copyright headers and fixed backwards - compatibility for an undocumented feature, for now. This - feature will later be removed and a new and documented - option will take its place.</p> - <p> - Own Id: OTP-7144</p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [httpd] - Error logs now has a pretty and a compact - format and access logs can be written on the common log - format or the extended common log format.</p> - <p> - Own Id: OTP-6661 Aux Id: Seq 7764 </p> - </item> - <item> - <p> - [httpc] - Added acceptance of missing reason phrase to - the relaxed mode.</p> - <p> - Own Id: OTP-7024</p> - </item> - <item> - <p> - [httpc] - A new option has been added to enable the - client to act as lower version clients, by default the - client is an HTTP/1.1 client.</p> - <p> - Own Id: OTP-7043</p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0.1</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - [httpd] - Deprecated function httpd:start/1 did not - accept all inputs that it had done previously. This - should now work again.</p> - <p> - Own Id: OTP-7040</p> - </item> - </list> - </section> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [httpd] - Changed validity check on bind_address so that - it uses inet:getaddr instead of inet:gethostbyaddr as the - former puts a too hard restriction on the bind_address.</p> - <p> - Own Id: OTP-7041 Aux Id: seq10829 </p> - </item> - <item> - <p> - [httpc] - Internal process now does try-catch and - terminates normally in case of HTTP parse errors. - Semantical the client works just as before returning an - error message to the client, even if the error massage - has been enhanced, but there is no supervisor report in - the shell of a internal process crashing. (Which was the - expected behavior and not a fault.)</p> - <p> - Own Id: OTP-7042</p> - </item> - </list> - </section> - - </section> - - <section><title>Inets 5.0</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - [httpd, httpc] - Deprecated base64 decode/encode - functions have been removed. Inets uses base64 in STDLIB - instead.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-6485</p> - </item> - <item> - <p> - [httpd] - It is now possible to restrict the length of - acceptable URI:s in the HTTP server.</p> - <p> - Own Id: OTP-6572</p> - </item> - <item> - <p> - [httpc] - Profiles are now supported i.e. the options - available in set_options/1 can be set locally for a - certain profile and do not have to affect all - HTTP-requests issued in the Erlang node. Calls to the - HTTP client API functions not using the profile argument - will use the default profile.</p> - <p> - Own Id: OTP-6690</p> - </item> - <item> - <p> - A new uniform Inets interface provides a flexible way to - start/stop Inets services and get information about - running services. See inets(3). This also means that - inflexibilities in the HTTP server has been removed and - more default values has been added.</p> - <p> - Own Id: OTP-6705</p> - </item> - <item> - <p> - [tftp] Logged errors have been changed to be logged - warnings.</p> - <p> - Own Id: OTP-6916 Aux Id: seq10737 </p> - </item> - <item> - <p> - [httpc] - The client will now return the proper value - when receiving a HTTP 204 code instead of hanging.</p> - <p> - Own Id: OTP-6982</p> - </item> - <item> - <p> - The Inets application now has to be explicitly started - and stopped i.e. it will not automatically be started as - a temporary application as it did before. Although a - practical feature when testing things in the shell it is - not desirable that people take advantage of this and not - start the Inets application in a correct way in their - products. Added functions to the Inets API that call - application:start/stop.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-6993</p> - </item> - </list> - </section> - - <!-- p>For information about older versions see - <url href="part_notes_history_frame.html">release notes history</url>.</p --> - </section> </chapter> diff --git a/lib/inets/doc/src/notes_history.xml b/lib/inets/doc/src/notes_history.xml index f70ce5cf46..151bec375e 100644 --- a/lib/inets/doc/src/notes_history.xml +++ b/lib/inets/doc/src/notes_history.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2010</year> + <year>2004</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index ac72963347..b6da92947c 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -55,9 +55,10 @@ -include("ftp_internal.hrl"). %% Constante used in internal state definition --define(CONNECTION_TIMEOUT, 60*1000). --define(DEFAULT_MODE, passive). --define(PROGRESS_DEFAULT, ignore). +-define(CONNECTION_TIMEOUT, 60*1000). +-define(DATA_ACCEPT_TIMEOUT, infinity). +-define(DEFAULT_MODE, passive). +-define(PROGRESS_DEFAULT, ignore). %% Internal Constants -define(FTP_PORT, 21). @@ -88,7 +89,8 @@ %% data needed further on. caller = undefined, % term() ipfamily, % inet | inet6 | inet6fb4 - progress = ignore % ignore | pid() + progress = ignore, % ignore | pid() + dtimeout = ?DATA_ACCEPT_TIMEOUT % non_neg_integer() | infinity }). @@ -847,6 +849,7 @@ start_options(Options) -> %% host %% port %% timeout +%% dtimeout %% progress open_options(Options) -> ?fcrt("open_options", [{options, Options}]), @@ -875,7 +878,12 @@ open_options(Options) -> (_) -> false end, ValidateTimeout = - fun(Timeout) when is_integer(Timeout) andalso (Timeout > 0) -> true; + fun(Timeout) when is_integer(Timeout) andalso (Timeout >= 0) -> true; + (_) -> false + end, + ValidateDTimeout = + fun(DTimeout) when is_integer(DTimeout) andalso (DTimeout >= 0) -> true; + (infinity) -> true; (_) -> false end, ValidateProgress = @@ -893,6 +901,7 @@ open_options(Options) -> {port, ValidatePort, false, ?FTP_PORT}, {ipfamily, ValidateIpFamily, false, inet}, {timeout, ValidateTimeout, false, ?CONNECTION_TIMEOUT}, + {dtimeout, ValidateDTimeout, false, ?DATA_ACCEPT_TIMEOUT}, {progress, ValidateProgress, false, ?PROGRESS_DEFAULT}], validate_options(Options, ValidOptions, []). @@ -1037,13 +1046,15 @@ handle_call({_, {open, ip_comm, Opts}}, From, State) -> Mode = key_search(mode, Opts, ?DEFAULT_MODE), Port = key_search(port, Opts, ?FTP_PORT), Timeout = key_search(timeout, Opts, ?CONNECTION_TIMEOUT), + DTimeout = key_search(dtimeout, Opts, ?DATA_ACCEPT_TIMEOUT), Progress = key_search(progress, Opts, ignore), IpFamily = key_search(ipfamily, Opts, inet), - + State2 = State#state{client = From, mode = Mode, progress = progress(Progress), - ipfamily = IpFamily}, + ipfamily = IpFamily, + dtimeout = DTimeout}, ?fcrd("handle_call(open) -> setup ctrl connection with", [{host, Host}, {port, Port}, {timeout, Timeout}]), @@ -1064,11 +1075,13 @@ handle_call({_, {open, ip_comm, Host, Opts}}, From, State) -> Mode = key_search(mode, Opts, ?DEFAULT_MODE), Port = key_search(port, Opts, ?FTP_PORT), Timeout = key_search(timeout, Opts, ?CONNECTION_TIMEOUT), + DTimeout = key_search(dtimeout, Opts, ?DATA_ACCEPT_TIMEOUT), Progress = key_search(progress, Opts, ignore), State2 = State#state{client = From, mode = Mode, - progress = progress(Progress)}, + progress = progress(Progress), + dtimeout = DTimeout}, case setup_ctrl_connection(Host, Port, Timeout, State2) of {ok, State3, WaitTimeout} -> @@ -1657,9 +1670,19 @@ handle_ctrl_result({pos_compl, Lines}, %%-------------------------------------------------------------------------- %% Directory listing handle_ctrl_result({pos_prel, _}, #state{caller = {dir, Dir}} = State) -> - NewState = accept_data_connection(State), - activate_data_connection(NewState), - {noreply, NewState#state{caller = {handle_dir_result, Dir}}}; + case accept_data_connection(State) of + {ok, NewState} -> + activate_data_connection(NewState), + {noreply, NewState#state{caller = {handle_dir_result, Dir}}}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_result, Dir, Data}, client = From} @@ -1756,9 +1779,19 @@ handle_ctrl_result({Status, _}, %%-------------------------------------------------------------------------- %% File handling - recv_bin handle_ctrl_result({pos_prel, _}, #state{caller = recv_bin} = State) -> - NewState = accept_data_connection(State), - activate_data_connection(NewState), - {noreply, NewState}; + case accept_data_connection(State) of + {ok, NewState} -> + activate_data_connection(NewState), + {noreply, NewState}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({pos_compl, _}, #state{caller = {recv_bin, Data}, client = From} = State) -> @@ -1780,16 +1813,37 @@ handle_ctrl_result({Status, _}, #state{caller = {recv_bin, _}} = State) -> handle_ctrl_result({pos_prel, _}, #state{client = From, caller = start_chunk_transfer} = State) -> - NewState = accept_data_connection(State), - gen_server:reply(From, ok), - {noreply, NewState#state{chunk = true, client = undefined, - caller = undefined}}; + case accept_data_connection(State) of + {ok, NewState} -> + gen_server:reply(From, ok), + {noreply, NewState#state{chunk = true, client = undefined, + caller = undefined}}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; + %%-------------------------------------------------------------------------- %% File handling - recv_file handle_ctrl_result({pos_prel, _}, #state{caller = {recv_file, _}} = State) -> - NewState = accept_data_connection(State), - activate_data_connection(NewState), - {noreply, NewState}; + case accept_data_connection(State) of + {ok, NewState} -> + activate_data_connection(NewState), + {noreply, NewState}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({Status, _}, #state{caller = {recv_file, Fd}} = State) -> file_close(Fd), @@ -1800,17 +1854,38 @@ handle_ctrl_result({Status, _}, #state{caller = {recv_file, Fd}} = State) -> %% File handling - transfer_* handle_ctrl_result({pos_prel, _}, #state{caller = {transfer_file, Fd}} = State) -> - NewState = accept_data_connection(State), - send_file(Fd, NewState); + case accept_data_connection(State) of + {ok, NewState} -> + send_file(Fd, NewState); + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; handle_ctrl_result({pos_prel, _}, #state{caller = {transfer_data, Bin}} = State) -> - NewState = accept_data_connection(State), - send_data_message(NewState, Bin), - close_data_connection(NewState), - activate_ctrl_connection(NewState), - {noreply, NewState#state{caller = transfer_data_second_phase, - dsock = undefined}}; + case accept_data_connection(State) of + {ok, NewState} -> + send_data_message(NewState, Bin), + close_data_connection(NewState), + activate_ctrl_connection(NewState), + {noreply, NewState#state{caller = transfer_data_second_phase, + dsock = undefined}}; + {error, _Reason} = ERROR -> + case State#state.client of + undefined -> + {stop, ERROR, State}; + From -> + gen_server:reply(From, ERROR), + {stop, normal, State#state{client = undefined}} + end + end; + %%-------------------------------------------------------------------------- %% Default handle_ctrl_result({Status, Lines}, #state{client = From} = State) @@ -2009,16 +2084,20 @@ connect2(Host, Port, IpFam, Timeout) -> Error end. - -accept_data_connection(#state{mode = active, - dsock = {lsock, LSock}} = State) -> - {ok, Socket} = gen_tcp:accept(LSock), - gen_tcp:close(LSock), - State#state{dsock = Socket}; +accept_data_connection(#state{mode = active, + dtimeout = DTimeout, + dsock = {lsock, LSock}} = State) -> + case gen_tcp:accept(LSock, DTimeout) of + {ok, Socket} -> + gen_tcp:close(LSock), + {ok, State#state{dsock = Socket}}; + {error, Reason} -> + {error, {data_connect_failed, Reason}} + end; accept_data_connection(#state{mode = passive} = State) -> - State. + {ok, State}. send_ctrl_message(#state{csock = Socket, verbose = Verbose}, Message) -> %% io:format("send control message: ~n~p~n", [lists:flatten(Message)]), diff --git a/lib/inets/src/http_client/Makefile b/lib/inets/src/http_client/Makefile index 0397b48ab2..3960c36d00 100644 --- a/lib/inets/src/http_client/Makefile +++ b/lib/inets/src/http_client/Makefile @@ -41,7 +41,6 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # Target Specs # ---------------------------------------------------- MODULES = \ - http \ httpc \ httpc_cookie \ httpc_handler \ diff --git a/lib/inets/src/http_client/http.erl b/lib/inets/src/http_client/http.erl deleted file mode 100644 index bbe2fec267..0000000000 --- a/lib/inets/src/http_client/http.erl +++ /dev/null @@ -1,132 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2002-2010. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% -%% - -%%% Description: OLD API MODULE - USE httpc INSTEAD - --module(http). - --deprecated({request, 1, next_major_release}). --deprecated({request, 2, next_major_release}). --deprecated({request, 4, next_major_release}). --deprecated({request, 5, next_major_release}). --deprecated({cancel_request, 1, next_major_release}). --deprecated({cancel_request, 2, next_major_release}). --deprecated({set_option, 2, next_major_release}). --deprecated({set_option, 3, next_major_release}). --deprecated({set_options, 1, next_major_release}). --deprecated({set_options, 2, next_major_release}). --deprecated({verify_cookies, 2, next_major_release}). --deprecated({verify_cookies, 3, next_major_release}). --deprecated({cookie_header, 1, next_major_release}). --deprecated({cookie_header, 2, next_major_release}). --deprecated({stream_next, 1, next_major_release}). --deprecated({default_profile, 0, next_major_release}). - -%% Deprecated --export([ - request/1, request/2, request/4, request/5, - cancel_request/1, cancel_request/2, - set_option/2, set_option/3, - set_options/1, set_options/2, - verify_cookies/2, verify_cookies/3, - cookie_header/1, cookie_header/2, - stream_next/1, - default_profile/0 - ]). - - -%%%========================================================================= -%%% API -%%%========================================================================= - -%%-------------------------------------------------------------------------- -%% request(Url [, Profile]) -> -%% request(Method, Request, HTTPOptions, Options [, Profile]) -%%-------------------------------------------------------------------------- - -request(Url) -> httpc:request(Url). -request(Url, Profile) -> httpc:request(Url, Profile). - -request(Method, Request, HttpOptions, Options) -> - httpc:request(Method, Request, HttpOptions, Options). -request(Method, Request, HttpOptions, Options, Profile) -> - httpc:request(Method, Request, HttpOptions, Options, Profile). - - -%%-------------------------------------------------------------------------- -%% cancel_request(RequestId [, Profile]) -%%------------------------------------------------------------------------- - -cancel_request(RequestId) -> - httpc:cancel_request(RequestId). -cancel_request(RequestId, Profile) -> - httpc:cancel_request(RequestId, Profile). - - -%%-------------------------------------------------------------------------- -%% set_options(Options [, Profile]) -%% set_option(Key, Value [, Profile]) -%%------------------------------------------------------------------------- - -set_options(Options) -> - httpc:set_options(Options). -set_options(Options, Profile) -> - httpc:set_options(Options, Profile). - -set_option(Key, Value) -> - httpc:set_option(Key, Value). -set_option(Key, Value, Profile) -> - httpc:set_option(Key, Value, Profile). - - -%%-------------------------------------------------------------------------- -%% verify_cookies(SetCookieHeaders, Url [, Profile]) -%%------------------------------------------------------------------------- - -verify_cookies(SetCookieHeaders, Url) -> - httpc:store_cookies(SetCookieHeaders, Url). -verify_cookies(SetCookieHeaders, Url, Profile) -> - httpc:store_cookies(SetCookieHeaders, Url, Profile). - - -%%-------------------------------------------------------------------------- -%% cookie_header(Url [, Profile]) -%%------------------------------------------------------------------------- - -cookie_header(Url) -> - httpc:cookie_header(Url). -cookie_header(Url, Profile) -> - httpc:cookie_header(Url, Profile). - - -%%-------------------------------------------------------------------------- -%% stream_next(Pid) -%%------------------------------------------------------------------------- - -stream_next(Pid) -> - httpc:stream_next(Pid). - - -%%-------------------------------------------------------------------------- -%% default_profile() -%%------------------------------------------------------------------------- - -default_profile() -> - httpc:default_profile(). diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index fe8e93af1f..d72c34fa6b 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -105,7 +105,6 @@ request(Url, Profile) -> %% {ssl, SSLOptions} | {proxy_auth, {User, Password}} %% Ssloptions = ssl_options() | %% {ssl, ssl_options()} | -%% {ossl, ssl_options()} | %% {essl, ssl_options()} %% ssl_options() = [ssl_option()] %% ssl_option() = {verify, code()} | @@ -142,7 +141,9 @@ request(Url, Profile) -> request(Method, Request, HttpOptions, Options) -> request(Method, Request, HttpOptions, Options, default_profile()). -request(Method, {Url, Headers}, HTTPOptions, Options, Profile) +request(Method, + {Url, Headers}, + HTTPOptions, Options, Profile) when (Method =:= options) orelse (Method =:= get) orelse (Method =:= head) orelse @@ -155,15 +156,17 @@ request(Method, {Url, Headers}, HTTPOptions, Options, Profile) {http_options, HTTPOptions}, {options, Options}, {profile, Profile}]), - case http_uri:parse(Url) of + case http_uri:parse(Url, Options) of {error, Reason} -> {error, Reason}; - ParsedUrl -> + {ok, ParsedUrl} -> handle_request(Method, Url, ParsedUrl, Headers, [], [], HTTPOptions, Options, Profile) end; -request(Method, {Url,Headers,ContentType,Body}, HTTPOptions, Options, Profile) +request(Method, + {Url, Headers, ContentType, Body}, + HTTPOptions, Options, Profile) when ((Method =:= post) orelse (Method =:= put)) andalso (is_atom(Profile) orelse is_pid(Profile)) -> ?hcrt("request", [{method, Method}, @@ -174,10 +177,10 @@ request(Method, {Url,Headers,ContentType,Body}, HTTPOptions, Options, Profile) {http_options, HTTPOptions}, {options, Options}, {profile, Profile}]), - case http_uri:parse(Url) of + case http_uri:parse(Url, Options) of {error, Reason} -> {error, Reason}; - ParsedUrl -> + {ok, ParsedUrl} -> handle_request(Method, Url, ParsedUrl, Headers, ContentType, Body, HTTPOptions, Options, Profile) @@ -268,7 +271,10 @@ store_cookies(SetCookieHeaders, Url, Profile) {profile, Profile}]), try begin - {_, _, Host, Port, Path, _} = http_uri:parse(Url), + %% Since the Address part is not actually used + %% by the manager when storing cookies, we dont + %% care about ipv6-host-with-brackets. + {ok, {_, _, Host, Port, Path, _}} = http_uri:parse(Url), Address = {Host, Port}, ProfileName = profile_name(Profile), Cookies = httpc_cookie:cookies(SetCookieHeaders, Path, Host), @@ -465,6 +471,8 @@ handle_request(Method, Url, HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), + BracketedHost = proplists:get_value(ipv6_host_with_brackets, + Options), MaybeEscPath = maybe_encode_uri(HTTPOptions, Path), MaybeEscQuery = maybe_encode_uri(HTTPOptions, Query), AbsUri = maybe_encode_uri(HTTPOptions, Url), @@ -483,7 +491,8 @@ handle_request(Method, Url, stream = Stream, headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, - started = Started}, + started = Started, + ipv6_host_with_brackets = BracketedHost}, case httpc_manager:request(Request, profile_name(Profile)) of {ok, RequestId} -> @@ -644,8 +653,6 @@ http_options_default() -> {ok, {?HTTP_DEFAULT_SSL_KIND, Value}}; ({ssl, SslOptions}) when is_list(SslOptions) -> {ok, {?HTTP_DEFAULT_SSL_KIND, SslOptions}}; - ({ossl, SslOptions}) when is_list(SslOptions) -> - {ok, {ossl, SslOptions}}; ({essl, SslOptions}) when is_list(SslOptions) -> {ok, {essl, SslOptions}}; (_) -> @@ -742,14 +749,17 @@ request_options_defaults() -> error end, + VerifyBrackets = VerifyBoolean, + [ - {sync, true, VerifySync}, - {stream, none, VerifyStream}, - {body_format, string, VerifyBodyFormat}, - {full_result, true, VerifyFullResult}, - {headers_as_is, false, VerifyHeaderAsIs}, - {receiver, self(), VerifyReceiver}, - {socket_opts, undefined, VerifySocketOpts} + {sync, true, VerifySync}, + {stream, none, VerifyStream}, + {body_format, string, VerifyBodyFormat}, + {full_result, true, VerifyFullResult}, + {headers_as_is, false, VerifyHeaderAsIs}, + {receiver, self(), VerifyReceiver}, + {socket_opts, undefined, VerifySocketOpts}, + {ipv6_host_with_brackets, false, VerifyBrackets} ]. request_options(Options) -> diff --git a/lib/inets/src/http_client/httpc_cookie.erl b/lib/inets/src/http_client/httpc_cookie.erl index d1f46c0b0b..69900bae65 100644 --- a/lib/inets/src/http_client/httpc_cookie.erl +++ b/lib/inets/src/http_client/httpc_cookie.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,12 +18,32 @@ %% %% Description: Cookie handling according to RFC 2109 +%% The syntax for the Set-Cookie response header is +%% +%% set-cookie = "Set-Cookie:" cookies +%% cookies = 1#cookie +%% cookie = NAME "=" VALUE *(";" cookie-av) +%% NAME = attr +%% VALUE = value +%% cookie-av = "Comment" "=" value +%% | "Domain" "=" value +%% | "Max-Age" "=" value +%% | "Path" "=" value +%% | "Secure" +%% | "Version" "=" 1*DIGIT + + +%% application:start(inets). +%% httpc:set_options([{cookies, enabled}, {proxy, {{"www-proxy.ericsson.se",8080}, ["*.ericsson.se"]}}]). +%% (catch httpc:request("http://www.expedia.com")). + -module(httpc_cookie). -include("httpc_internal.hrl"). -export([open_db/3, close_db/1, insert/2, header/4, cookies/3]). -export([reset_db/1, which_cookies/1]). +-export([image_of/2, print/2]). -record(cookie_db, {db, session_db}). @@ -125,7 +145,7 @@ insert(#cookie_db{db = Db} = CookieDb, name = Name, path = Path, max_age = 0}) -> - ?hcrt("insert", [{domain, Key}, {name, Name}, {path, Path}]), + ?hcrt("insert cookie", [{domain, Key}, {name, Name}, {path, Path}]), Pattern = #http_cookie{domain = Key, name = Name, path = Path, _ = '_'}, case dets:match_object(Db, Pattern) of [] -> @@ -136,7 +156,7 @@ insert(#cookie_db{db = Db} = CookieDb, ok; insert(#cookie_db{db = Db} = CookieDb, #http_cookie{domain = Key, name = Name, path = Path} = Cookie) -> - ?hcrt("insert", [{cookie, Cookie}]), + ?hcrt("insert cookie", [{cookie, Cookie}]), Pattern = #http_cookie{domain = Key, name = Name, path = Path, @@ -163,6 +183,7 @@ header(CookieDb, Scheme, {Host, _}, Path) -> [] -> {"cookie", ""}; Cookies -> + %% print_cookies("Header Cookies", Cookies), {"cookie", cookies_to_string(Scheme, Cookies)} end. @@ -173,11 +194,20 @@ header(CookieDb, Scheme, {Host, _}, Path) -> %%-------------------------------------------------------------------- cookies(Headers, RequestPath, RequestHost) -> + ?hcrt("cookies", [{headers, Headers}, {request_path, RequestPath}, {request_host, RequestHost}]), + Cookies = parse_set_cookies(Headers, {RequestPath, RequestHost}), - accept_cookies(Cookies, RequestPath, RequestHost). + + %% print_cookies("Parsed Cookies", Cookies), + + AcceptedCookies = accept_cookies(Cookies, RequestPath, RequestHost), + + %% print_cookies("Accepted Cookies", AcceptedCookies), + + AcceptedCookies. %%-------------------------------------------------------------------- @@ -266,7 +296,8 @@ cookies_to_string(_, [], CookieStrs) -> lists:flatten(lists:reverse(CookieStrs)) end; -cookies_to_string(https, [#http_cookie{secure = true} = Cookie| Cookies], +cookies_to_string(https = Scheme, + [#http_cookie{secure = true} = Cookie| Cookies], CookieStrs) -> Str = case Cookies of [] -> @@ -274,7 +305,7 @@ cookies_to_string(https, [#http_cookie{secure = true} = Cookie| Cookies], _ -> cookie_to_string(Cookie) ++ "; " end, - cookies_to_string(https, Cookies, [Str | CookieStrs]); + cookies_to_string(Scheme, Cookies, [Str | CookieStrs]); cookies_to_string(Scheme, [#http_cookie{secure = true}| Cookies], CookieStrs) -> @@ -303,63 +334,54 @@ add_domain(Str, #http_cookie{domain_default = true}) -> add_domain(Str, #http_cookie{domain = Domain}) -> Str ++ "; $Domain=" ++ Domain. -parse_set_cookies(OtherHeaders, DefaultPathDomain) -> - SetCookieHeaders = - lists:foldl(fun({"set-cookie", Value}, Acc) -> - [string:tokens(Value, ",")| Acc]; - (_, Acc) -> - Acc - end, [], OtherHeaders), - - lists:flatten( - lists:map(fun(CookieHeader) -> - NewHeader = fix_netscape_cookie(CookieHeader, []), - parse_set_cookie(NewHeader, [], DefaultPathDomain) - end, - SetCookieHeaders)). - -parse_set_cookie([], AccCookies, _) -> - AccCookies; -parse_set_cookie([CookieHeader | CookieHeaders], AccCookies, - Defaults = {DefaultPath, DefaultDomain}) -> - [CookieStr | Attributes] = case string:tokens(CookieHeader, ";") of - [CStr] -> - [CStr, ""]; - [CStr | Attr] -> - [CStr, Attr] - end, - Pos = string:chr(CookieStr, $=), - Name = string:substr(CookieStr, 1, Pos - 1), - Value = string:substr(CookieStr, Pos + 1), - Cookie = #http_cookie{name = string:strip(Name), - value = string:strip(Value)}, - NewAttributes = parse_set_cookie_attributes(Attributes), - TmpCookie = cookie_attributes(NewAttributes, Cookie), +parse_set_cookies(CookieHeaders, DefaultPathDomain) -> + SetCookieHeaders = [Value || {"set-cookie", Value} <- CookieHeaders], + Cookies = [parse_set_cookie(SetCookieHeader, DefaultPathDomain) || + SetCookieHeader <- SetCookieHeaders], + %% print_cookies("Parsed Cookies", Cookies), + Cookies. + +parse_set_cookie(CookieHeader, {DefaultPath, DefaultDomain}) -> + %% io:format("Raw Cookie: ~s~n", [CookieHeader]), + Pos = string:chr(CookieHeader, $=), + Name = string:substr(CookieHeader, 1, Pos - 1), + {Value, Attrs} = + case string:substr(CookieHeader, Pos + 1) of + [$;|ValueAndAttrs] -> + {"", string:tokens(ValueAndAttrs, ";")}; + ValueAndAttrs -> + [V | A] = string:tokens(ValueAndAttrs, ";"), + {V, A} + end, + Cookie = #http_cookie{name = string:strip(Name), + value = string:strip(Value)}, + Attributes = parse_set_cookie_attributes(Attrs), + TmpCookie = cookie_attributes(Attributes, Cookie), %% Add runtime defult values if necessary - NewCookie = domain_default(path_default(TmpCookie, DefaultPath), - DefaultDomain), - parse_set_cookie(CookieHeaders, [NewCookie | AccCookies], Defaults). - -parse_set_cookie_attributes([]) -> - []; -parse_set_cookie_attributes([Attributes]) -> - lists:map(fun(Attr) -> - [AttrName, AttrValue] = - case string:tokens(Attr, "=") of - %% All attributes have the form - %% Name=Value except "secure"! - [Name] -> - [Name, ""]; - [Name, Value] -> - [Name, Value]; - %% Anything not expected will be - %% disregarded - _ -> - ["Dummy",""] - end, - {http_util:to_lower(string:strip(AttrName)), - string:strip(AttrValue)} - end, Attributes). + NewCookie = domain_default(path_default(TmpCookie, DefaultPath), + DefaultDomain), + NewCookie. + +parse_set_cookie_attributes(Attributes) when is_list(Attributes) -> + [parse_set_cookie_attribute(A) || A <- Attributes]. + +parse_set_cookie_attribute(Attribute) -> + {AName, AValue} = + case string:tokens(Attribute, "=") of + %% All attributes have the form + %% Name=Value except "secure"! + [Name] -> + {Name, ""}; + [Name, Value] -> + {Name, Value}; + %% Anything not expected will be + %% disregarded + _ -> + {"Dummy", ""} + end, + StrippedName = http_util:to_lower(string:strip(AName)), + StrippedValue = string:strip(AValue), + {StrippedName, StrippedValue}. cookie_attributes([], Cookie) -> Cookie; @@ -375,10 +397,15 @@ cookie_attributes([{"max-age", Value}| Attributes], Cookie) -> Cookie#http_cookie{max_age = ExpireTime}); %% Backwards compatibility with netscape cookies cookie_attributes([{"expires", Value}| Attributes], Cookie) -> - Time = http_util:convert_netscapecookie_date(Value), - ExpireTime = calendar:datetime_to_gregorian_seconds(Time), - cookie_attributes(Attributes, - Cookie#http_cookie{max_age = ExpireTime}); + try http_util:convert_netscapecookie_date(Value) of + Time -> + ExpireTime = calendar:datetime_to_gregorian_seconds(Time), + cookie_attributes(Attributes, + Cookie#http_cookie{max_age = ExpireTime}) + catch + _:_ -> + cookie_attributes(Attributes, Cookie) + end; cookie_attributes([{"path", Value}| Attributes], Cookie) -> cookie_attributes(Attributes, Cookie#http_cookie{path = Value}); @@ -476,20 +503,43 @@ path_sort(Cookies)-> lists:reverse(lists:keysort(#http_cookie.path, Cookies)). -%% Informally, the Set-Cookie response header comprises the token -%% Set-Cookie:, followed by a comma-separated list of one or more -%% cookies. Netscape cookies expires attribute may also have a, -%% in this case the header list will have been incorrectly split -%% in parse_set_cookies/2 this functions fix that problem. -fix_netscape_cookie([Cookie1, Cookie2 | Rest], Acc) -> - case inets_regexp:match(string:to_lower(Cookie1), "expires=") of - {_, _, _} -> - fix_netscape_cookie(Rest, [Cookie1 ++ Cookie2 | Acc]); - nomatch -> - fix_netscape_cookie([Cookie2 |Rest], [Cookie1| Acc]) - end; -fix_netscape_cookie([Cookie | Rest], Acc) -> - fix_netscape_cookie(Rest, [Cookie | Acc]); - -fix_netscape_cookie([], Acc) -> - Acc. +%% print_cookies(Header, Cookies) -> +%% io:format("~s:~n", [Header]), +%% Prefix = " ", +%% lists:foreach(fun(Cookie) -> print(Prefix, Cookie) end, Cookies). + +image_of(Prefix, + #http_cookie{domain = Domain, + domain_default = DomainDef, + name = Name, + value = Value, + comment = Comment, + max_age = MaxAge, + path = Path, + path_default = PathDef, + secure = Sec, + version = Version}) -> + lists:flatten( + io_lib:format("~sCookie ~s: " + "~n~s Value: ~p" + "~n~s Domain: ~p" + "~n~s DomainDef: ~p" + "~n~s Comment: ~p" + "~n~s MaxAge: ~p" + "~n~s Path: ~p" + "~n~s PathDef: ~p" + "~n~s Secure: ~p" + "~n~s Version: ~p", + [Prefix, Name, + Prefix, Value, + Prefix, Domain, + Prefix, DomainDef, + Prefix, Comment, + Prefix, MaxAge, + Prefix, Path, + Prefix, PathDef, + Prefix, Sec, + Prefix, Version])). + +print(Prefix, Cookie) when is_record(Cookie, http_cookie) -> + io:format("~s~n", [image_of(Prefix, Cookie)]). diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl index 1d8a5b6a92..e4127d992d 100644 --- a/lib/inets/src/http_client/httpc_internal.hrl +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -90,25 +90,28 @@ %%% All data associated to a specific HTTP request -record(request, { - id, % ref() - Request Id - from, % pid() - Caller - redircount = 0,% Number of redirects made for this request - scheme, % http | https - address, % ({Host,Port}) Destination Host and Port - path, % string() - Path of parsed URL - pquery, % string() - Rest of parsed URL - method, % atom() - HTTP request Method - headers, % #http_request_h{} - content, % {ContentType, Body} - Current HTTP request - settings, % #http_options{} - User defined settings - abs_uri, % string() ex: "http://www.erlang.org" - userinfo, % string() - optinal "<userinfo>@<host>:<port>" - stream, % Boolean() - stream async reply? - headers_as_is, % Boolean() - workaround for servers that does - % not honor the http standard, can also be used for testing purposes. - started, % integer() > 0 - When we started processing the request - timer, % undefined | ref() - socket_opts % undefined | [socket_option()] + id, % ref() - Request Id + from, % pid() - Caller + redircount = 0,% Number of redirects made for this request + scheme, % http | https + address, % ({Host,Port}) Destination Host and Port + path, % string() - Path of parsed URL + pquery, % string() - Rest of parsed URL + method, % atom() - HTTP request Method + headers, % #http_request_h{} + content, % {ContentType, Body} - Current HTTP request + settings, % #http_options{} - User defined settings + abs_uri, % string() ex: "http://www.erlang.org" + userinfo, % string() - optinal "<userinfo>@<host>:<port>" + stream, % boolean() - stream async reply? + headers_as_is, % boolean() - workaround for servers that does + % not honor the http standard, can also be used + % for testing purposes. + started, % integer() > 0 - When we started processing the + % request + timer, % undefined | ref() + socket_opts, % undefined | [socket_option()] + ipv6_host_with_brackets % boolean() } ). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index 9015bf1ce2..ab575d867e 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -256,19 +256,27 @@ reset_cookies(ProfileName) -> %%-------------------------------------------------------------------- -%% Function: which_cookies(Url, ProfileName) -> [cookie()] +%% Function: which_cookies(ProfileName) -> [cookie()] +%% which_cookies(Url, ProfileName) -> [cookie()] +%% which_cookies(Url, Options, ProfileName) -> [cookie()] %% %% Url = string() +%% Options = [option()] %% ProfileName = atom() +%% option() = {ipv6_host_with_brackets, boolean()} %% %% Description: Retrieves the cookies that would be sent when %% requesting <Url>. %%-------------------------------------------------------------------- -which_cookies(ProfileName) -> +which_cookies(ProfileName) when is_atom(ProfileName) -> call(ProfileName, which_cookies). -which_cookies(Url, ProfileName) -> - call(ProfileName, {which_cookies, Url}). +which_cookies(Url, ProfileName) + when is_list(Url) andalso is_atom(ProfileName) -> + call(ProfileName, {which_cookies, Url, []}). +which_cookies(Url, Options, ProfileName) + when is_list(Url) andalso is_list(Options) andalso is_atom(ProfileName) -> + call(ProfileName, {which_cookies, Url, Options}). %%-------------------------------------------------------------------- @@ -395,15 +403,16 @@ handle_call(which_cookies, _, #state{cookie_db = CookieDb} = State) -> CookieHeaders = httpc_cookie:which_cookies(CookieDb), {reply, CookieHeaders, State}; -handle_call({which_cookies, Url}, _, #state{cookie_db = CookieDb} = State) -> - ?hcrv("which cookies", [{url, Url}]), - case http_uri:parse(Url) of - {Scheme, _, Host, Port, Path, _} -> +handle_call({which_cookies, Url, Options}, _, + #state{cookie_db = CookieDb} = State) -> + ?hcrv("which cookies", [{url, Url}, {options, Options}]), + case http_uri:parse(Url, Options) of + {ok, {Scheme, _, Host, Port, Path, _}} -> CookieHeaders = httpc_cookie:header(CookieDb, Scheme, {Host, Port}, Path), {reply, CookieHeaders, State}; - Msg -> - {reply, Msg, State} + {error, _} = ERROR -> + {reply, ERROR, State} end; handle_call(info, _, State) -> diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 207b96271c..2414ed0911 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -340,7 +340,9 @@ redirect(Response = {StatusLine, Headers, Body}, Request) -> undefined -> transparent(Response, Request); RedirUrl -> - case http_uri:parse(RedirUrl) of + UrlParseOpts = [{ipv6_host_with_brackets, + Request#request.ipv6_host_with_brackets}], + case http_uri:parse(RedirUrl, UrlParseOpts) of {error, no_scheme} when (Request#request.settings)#http_options.relaxed -> NewLocation = fix_relative_uri(Request, RedirUrl), @@ -350,10 +352,9 @@ redirect(Response = {StatusLine, Headers, Body}, Request) -> {error, Reason} -> {ok, error(Request, Reason), Data}; %% Automatic redirection - {Scheme, _, Host, Port, Path, Query} -> + {ok, {Scheme, _, Host, Port, Path, Query}} -> NewHeaders = - (Request#request.headers)#http_request_h{host = - Host}, + (Request#request.headers)#http_request_h{host = Host}, NewRequest = Request#request{redircount = Request#request.redircount+1, diff --git a/lib/inets/src/http_lib/http_internal.hrl b/lib/inets/src/http_lib/http_internal.hrl index 2e924667c6..97cf474ab9 100644 --- a/lib/inets/src/http_lib/http_internal.hrl +++ b/lib/inets/src/http_lib/http_internal.hrl @@ -28,7 +28,6 @@ -define(HTTP_MAX_URI_SIZE, nolimit). -ifndef(HTTP_DEFAULT_SSL_KIND). -%% -define(HTTP_DEFAULT_SSL_KIND, ossl). -define(HTTP_DEFAULT_SSL_KIND, essl). -endif. % -ifdef(HTTP_DEFAULT_SSL_KIND). diff --git a/lib/inets/src/http_lib/http_transport.erl b/lib/inets/src/http_lib/http_transport.erl index 9b8190ebed..5eb827032f 100644 --- a/lib/inets/src/http_lib/http_transport.erl +++ b/lib/inets/src/http_lib/http_transport.erl @@ -62,8 +62,6 @@ start(ip_comm) -> %% This is just for backward compatibillity start({ssl, _}) -> do_start_ssl(); -start({ossl, _}) -> - do_start_ssl(); start({essl, _}) -> do_start_ssl(). @@ -126,22 +124,6 @@ connect(ip_comm = _SocketType, {Host, Port}, Opts0, Timeout) connect({ssl, SslConfig}, Address, Opts, Timeout) -> connect({?HTTP_DEFAULT_SSL_KIND, SslConfig}, Address, Opts, Timeout); -connect({ossl, SslConfig}, {Host, Port}, _, Timeout) -> - Opts = [binary, {active, false}, {ssl_imp, old}] ++ SslConfig, - ?hlrt("connect using ossl", - [{host, Host}, - {port, Port}, - {ssl_config, SslConfig}, - {timeout, Timeout}]), - case (catch ssl:connect(Host, Port, Opts, Timeout)) of - {'EXIT', Reason} -> - {error, {eoptions, Reason}}; - {ok, _} = OK -> - OK; - {error, _} = ERROR -> - ERROR - end; - connect({essl, SslConfig}, {Host, Port}, Opts0, Timeout) -> Opts = [binary, {active, false}, {ssl_imp, new} | Opts0] ++ SslConfig, ?hlrt("connect using essl", @@ -187,13 +169,6 @@ listen({ssl, SSLConfig}, Addr, Port) -> {ssl_config, SSLConfig}]), listen({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Addr, Port); -listen({ossl, SSLConfig}, Addr, Port) -> - ?hlrt("listen (ossl)", - [{addr, Addr}, - {port, Port}, - {ssl_config, SSLConfig}]), - listen_ssl(Addr, Port, [{ssl_imp, old} | SSLConfig]); - listen({essl, SSLConfig}, Addr, Port) -> ?hlrt("listen (essl)", [{addr, Addr}, @@ -353,8 +328,6 @@ accept(ip_comm, ListenSocket, Timeout) -> accept({ssl, SSLConfig}, ListenSocket, Timeout) -> accept({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, ListenSocket, Timeout); -accept({ossl, _SSLConfig}, ListenSocket, Timeout) -> - ssl:transport_accept(ListenSocket, Timeout); accept({essl, _SSLConfig}, ListenSocket, Timeout) -> ssl:transport_accept(ListenSocket, Timeout). @@ -374,9 +347,6 @@ controlling_process(ip_comm, Socket, NewOwner) -> controlling_process({ssl, SSLConfig}, Socket, NewOwner) -> controlling_process({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket, NewOwner); -controlling_process({ossl, _}, Socket, NewOwner) -> - ssl:controlling_process(Socket, NewOwner); - controlling_process({essl, _}, Socket, NewOwner) -> ssl:controlling_process(Socket, NewOwner). @@ -397,13 +367,6 @@ setopts(ip_comm, Socket, Options) -> setopts({ssl, SSLConfig}, Socket, Options) -> setopts({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket, Options); -setopts({ossl, _}, Socket, Options) -> - ?hlrt("[o]ssl setopts", [{socket, Socket}, {options, Options}]), - Reason = (catch ssl:setopts(Socket, Options)), - ?hlrt("[o]ssl setopts result", [{reason, Reason}]), - Reason; - - setopts({essl, _}, Socket, Options) -> ?hlrt("[e]ssl setopts", [{socket, Socket}, {options, Options}]), Reason = (catch ssl:setopts(Socket, Options)), @@ -435,10 +398,6 @@ getopts(ip_comm, Socket, Options) -> getopts({ssl, SSLConfig}, Socket, Options) -> getopts({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket, Options); -getopts({ossl, _}, Socket, Options) -> - ?hlrt("ssl getopts", [{socket, Socket}, {options, Options}]), - getopts_ssl(Socket, Options); - getopts({essl, _}, Socket, Options) -> ?hlrt("essl getopts", [{socket, Socket}, {options, Options}]), getopts_ssl(Socket, Options). @@ -472,9 +431,6 @@ getstat(ip_comm = _SocketType, Socket) -> getstat({ssl, SSLConfig}, Socket) -> getstat({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket); -getstat({ossl, _} = _SocketType, _Socket) -> - []; - getstat({essl, _} = _SocketType, _Socket) -> []. @@ -493,9 +449,6 @@ send(ip_comm, Socket, Message) -> send({ssl, SSLConfig}, Socket, Message) -> send({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket, Message); -send({ossl, _}, Socket, Message) -> - ssl:send(Socket, Message); - send({essl, _}, Socket, Message) -> ssl:send(Socket, Message). @@ -514,9 +467,6 @@ close(ip_comm, Socket) -> close({ssl, SSLConfig}, Socket) -> close({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket); -close({ossl, _}, Socket) -> - ssl:close(Socket); - close({essl, _}, Socket) -> ssl:close(Socket). @@ -538,9 +488,6 @@ peername(ip_comm, Socket) -> peername({ssl, SSLConfig}, Socket) -> peername({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket); -peername({ossl, _}, Socket) -> - do_peername(ssl:peername(Socket)); - peername({essl, _}, Socket) -> do_peername(ssl:peername(Socket)). @@ -573,9 +520,6 @@ sockname(ip_comm, Socket) -> sockname({ssl, SSLConfig}, Socket) -> sockname({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket); -sockname({ossl, _}, Socket) -> - do_sockname(ssl:sockname(Socket)); - sockname({essl, _}, Socket) -> do_sockname(ssl:sockname(Socket)). @@ -651,9 +595,6 @@ negotiate(ip_comm,_,_) -> negotiate({ssl, SSLConfig}, Socket, Timeout) -> ?hlrt("negotiate(ssl)", []), negotiate({?HTTP_DEFAULT_SSL_KIND, SSLConfig}, Socket, Timeout); -negotiate({ossl, _}, Socket, Timeout) -> - ?hlrt("negotiate(ossl)", []), - negotiate_ssl(Socket, Timeout); negotiate({essl, _}, Socket, Timeout) -> ?hlrt("negotiate(essl)", []), negotiate_ssl(Socket, Timeout). diff --git a/lib/inets/src/http_lib/http_uri.erl b/lib/inets/src/http_lib/http_uri.erl index 44b9face0b..32c6305a79 100644 --- a/lib/inets/src/http_lib/http_uri.erl +++ b/lib/inets/src/http_lib/http_uri.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2010. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,23 +16,30 @@ %% %% %CopyrightEnd% %% -%% +%% +%% RFC 3986 +%% -module(http_uri). --export([parse/1, encode/1, decode/1]). +-export([parse/1, parse/2, + encode/1, decode/1]). + %%%========================================================================= %%% API %%%========================================================================= parse(AbsURI) -> + parse(AbsURI, []). + +parse(AbsURI, Opts) -> case parse_scheme(AbsURI) of {error, Reason} -> {error, Reason}; {Scheme, Rest} -> - case (catch parse_uri_rest(Scheme, Rest)) of + case (catch parse_uri_rest(Scheme, Rest, Opts)) of {UserInfo, Host, Port, Path, Query} -> - {Scheme, UserInfo, Host, Port, Path, Query}; + {ok, {Scheme, UserInfo, Host, Port, Path, Query}}; _ -> {error, {malformed_url, AbsURI}} end @@ -42,35 +49,38 @@ encode(URI) -> Reserved = sets:from_list([$;, $:, $@, $&, $=, $+, $,, $/, $?, $#, $[, $], $<, $>, $\", ${, $}, $|, $\\, $', $^, $%, $ ]), - lists:append(lists:map(fun(Char) -> - uri_encode(Char, Reserved) - end, URI)). - -decode([$%,Hex1,Hex2|Rest]) -> - [hex2dec(Hex1)*16+hex2dec(Hex2)|decode(Rest)]; -decode([First|Rest]) -> - [First|decode(Rest)]; -decode([]) -> + %% lists:append(lists:map(fun(Char) -> uri_encode(Char, Reserved) end, URI)). + lists:append([uri_encode(Char, Reserved) || Char <- URI]). + +decode(String) -> + do_decode(String). + +do_decode([$%,Hex1,Hex2|Rest]) -> + [hex2dec(Hex1)*16+hex2dec(Hex2)|do_decode(Rest)]; +do_decode([First|Rest]) -> + [First|do_decode(Rest)]; +do_decode([]) -> []. + %%%======================================================================== %%% Internal functions %%%======================================================================== + parse_scheme(AbsURI) -> case split_uri(AbsURI, ":", {error, no_scheme}, 1, 1) of {error, no_scheme} -> {error, no_scheme}; {StrScheme, Rest} -> case list_to_atom(http_util:to_lower(StrScheme)) of - Scheme when Scheme == http; Scheme == https -> + Scheme when (Scheme =:= http) orelse (Scheme =:= https) -> {Scheme, Rest}; Scheme -> {error, {not_supported_scheme, Scheme}} end end. -parse_uri_rest(Scheme, "//" ++ URIPart) -> - +parse_uri_rest(Scheme, "//" ++ URIPart, Opts) -> {Authority, PathQuery} = case split_uri(URIPart, "/", URIPart, 1, 0) of Split = {_, _} -> @@ -85,8 +95,8 @@ parse_uri_rest(Scheme, "//" ++ URIPart) -> end, {UserInfo, HostPort} = split_uri(Authority, "@", {"", Authority}, 1, 1), - {Host, Port} = parse_host_port(Scheme, HostPort), - {Path, Query} = parse_path_query(PathQuery), + {Host, Port} = parse_host_port(Scheme, HostPort, Opts), + {Path, Query} = parse_path_query(PathQuery), {UserInfo, Host, Port, Path, Query}. @@ -94,13 +104,14 @@ parse_path_query(PathQuery) -> {Path, Query} = split_uri(PathQuery, "\\?", {PathQuery, ""}, 1, 0), {path(Path), Query}. -parse_host_port(Scheme,"[" ++ HostPort) -> %ipv6 +parse_host_port(Scheme,"[" ++ HostPort, Opts) -> %ipv6 DefaultPort = default_port(Scheme), {Host, ColonPort} = split_uri(HostPort, "\\]", {HostPort, ""}, 1, 1), + Host2 = maybe_ipv6_host_with_brackets(Host, Opts), {_, Port} = split_uri(ColonPort, ":", {"", DefaultPort}, 0, 1), - {Host, int_port(Port)}; + {Host2, int_port(Port)}; -parse_host_port(Scheme, HostPort) -> +parse_host_port(Scheme, HostPort, _Opts) -> DefaultPort = default_port(Scheme), {Host, Port} = split_uri(HostPort, ":", {HostPort, DefaultPort}, 1, 1), {Host, int_port(Port)}. @@ -114,6 +125,14 @@ split_uri(UriPart, SplitChar, NoMatchResult, SkipLeft, SkipRight) -> NoMatchResult end. +maybe_ipv6_host_with_brackets(Host, Opts) -> + case lists:keysearch(ipv6_host_with_brackets, 1, Opts) of + {value, {ipv6_host_with_brackets, true}} -> + "[" ++ Host ++ "]"; + _ -> + Host + end. + default_port(http) -> 80; default_port(https) -> diff --git a/lib/inets/src/http_lib/http_util.erl b/lib/inets/src/http_lib/http_util.erl index 5511ed388d..5b21170b78 100644 --- a/lib/inets/src/http_lib/http_util.erl +++ b/lib/inets/src/http_lib/http_util.erl @@ -104,6 +104,22 @@ convert_netscapecookie_date([_D,_A,_Y, $ , Sec = list_to_integer([S1,S2]), {{Year,Month,Day},{Hour,Min,Sec}}; +%% Example: Tue Jan 01 08:00:01 2036 GMT +convert_netscapecookie_date([_D,_A,_Y, $ , + M,O,N, $ , + D1,D2, $ , + H1,H2, $:, + M1,M2, $:, + S1,S2, $ , + Y1,Y2,Y3,Y4, $ |_Rest]) -> + Year = list_to_integer([Y1,Y2,Y3,Y4]), + Day = list_to_integer([D1,D2]), + Month = convert_month([M,O,N]), + Hour = list_to_integer([H1,H2]), + Min = list_to_integer([M1,M2]), + Sec = list_to_integer([S1,S2]), + {{Year,Month,Day},{Hour,Min,Sec}}; + %% Sloppy... convert_netscapecookie_date([_D,_A,_Y, $,, _SP, D1,D2,_DA, @@ -190,9 +206,7 @@ timeout(Timeout, Started) -> html_encode(Chars) -> Reserved = sets:from_list([$&, $<, $>, $\", $', $/]), - lists:append(lists:map(fun(Char) -> - char_to_html_entity(Char, Reserved) - end, Chars)). + lists:append([char_to_html_entity(Char, Reserved) || Char <- Chars]). %%%======================================================================== diff --git a/lib/inets/src/http_server/httpd_conf.erl b/lib/inets/src/http_server/httpd_conf.erl index d1b1ea0e14..7646300409 100644 --- a/lib/inets/src/http_server/httpd_conf.erl +++ b/lib/inets/src/http_server/httpd_conf.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -219,9 +219,8 @@ load("ServerName " ++ ServerName, []) -> load("SocketType " ++ SocketType, []) -> %% ssl is the same as HTTP_DEFAULT_SSL_KIND - %% ossl is ssl based on OpenSSL (the "old" ssl) %% essl is the pure Erlang-based ssl (the "new" ssl) - case check_enum(clean(SocketType), ["ssl", "ossl", "essl", "ip_comm"]) of + case check_enum(clean(SocketType), ["ssl", "essl", "ip_comm"]) of {ok, ValidSocketType} -> {ok, [], {socket_type, ValidSocketType}}; {error,_} -> @@ -541,7 +540,6 @@ validate_config_params([{server_name, Value} | _]) -> validate_config_params([{socket_type, Value} | Rest]) when (Value =:= ip_comm) orelse (Value =:= ssl) orelse - (Value =:= ossl) orelse (Value =:= essl) -> validate_config_params(Rest); validate_config_params([{socket_type, Value} | _]) -> @@ -811,7 +809,7 @@ lookup_socket_type(ConfigDB) -> case httpd_util:lookup(ConfigDB, socket_type, ip_comm) of ip_comm -> ip_comm; - SSL when (SSL =:= ssl) orelse (SSL =:= ossl) orelse (SSL =:= essl) -> + SSL when (SSL =:= ssl) orelse (SSL =:= essl) -> SSLTag = if (SSL =:= ssl) -> diff --git a/lib/inets/src/http_server/httpd_esi.erl b/lib/inets/src/http_server/httpd_esi.erl index aac5645282..000874d0a3 100644 --- a/lib/inets/src/http_server/httpd_esi.erl +++ b/lib/inets/src/http_server/httpd_esi.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/inets/src/http_server/httpd_file.erl b/lib/inets/src/http_server/httpd_file.erl index e8a8ab6411..f2ba33099e 100644 --- a/lib/inets/src/http_server/httpd_file.erl +++ b/lib/inets/src/http_server/httpd_file.erl @@ -36,9 +36,9 @@ handle_error(emfile, Op, _ModData, Path) -> handle_error(500, Op, none, Path, ": Too many open files"); handle_error({enfile,_}, Op, _ModData, Path) -> handle_error(500, Op, none, Path, ": File table overflow"); -handle_error(_Reason, Op, ModData, Path) -> - handle_error(404, Op, ModData, Path, ": File not found"). - +handle_error(_Reason, Op, _ModData, Path) -> + handle_error(500, Op, none, Path, ""). + handle_error(StatusCode, Op, none, Path, Reason) -> {StatusCode, none, ?NICE("Can't " ++ Op ++ " " ++ Path ++ Reason)}; handle_error(StatusCode, Op, ModData, Path, Reason) -> diff --git a/lib/inets/src/http_server/httpd_request.erl b/lib/inets/src/http_server/httpd_request.erl index 7084d9824a..5ba79b2706 100644 --- a/lib/inets/src/http_server/httpd_request.erl +++ b/lib/inets/src/http_server/httpd_request.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -309,12 +309,12 @@ validate_uri(RequestURI) -> (catch http_uri:decode(string:left(RequestURI, Ndx))) end, case UriNoQueryNoHex of - {'EXIT',_Reason} -> + {'EXIT', _Reason} -> {error, {bad_request, {malformed_syntax, RequestURI}}}; _ -> - Path = format_request_uri(UriNoQueryNoHex), - Path2=[X||X<-string:tokens(Path, "/"),X=/="."], %% OTP-5938 - validate_path( Path2,0, RequestURI) + Path = format_request_uri(UriNoQueryNoHex), + Path2 = [X||X<-string:tokens(Path, "/"),X=/="."], %% OTP-5938 + validate_path(Path2, 0, RequestURI) end. validate_path([], _, _) -> diff --git a/lib/inets/src/http_server/httpd_request_handler.erl b/lib/inets/src/http_server/httpd_request_handler.erl index c3b47ce390..d2f22fce93 100644 --- a/lib/inets/src/http_server/httpd_request_handler.erl +++ b/lib/inets/src/http_server/httpd_request_handler.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 1997-2011. All Rights Reserved. -%% +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the @@ -355,7 +355,7 @@ handle_http_msg({Method, Uri, Version, {RecordHeaders, Headers}, Body}, Reason = io_lib:format("Forbidden URI: ~p~n", [URI]), error_log(Reason, ModData), {stop, normal, State#state{response_sent = true}}; - {error,{bad_request, {malformed_syntax, URI}}} -> + {error, {bad_request, {malformed_syntax, URI}}} -> ?hdrd("validation failed: bad request - malformed syntax", [{uri, URI}]), httpd_response:send_status(ModData#mod{http_version = Version}, diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index ea9cfbf4f2..dd7223876e 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -78,6 +78,7 @@ traverse_modules(ModData,[Module|Rest]) -> [Module, Reason])), report_error(mod_log, ModData#mod.config_db, String), report_error(mod_disk_log, ModData#mod.config_db, String), + send_status(ModData, 500, none), done; done -> ?hdrt("traverse modules - done", []), @@ -100,12 +101,19 @@ send_status(#mod{socket_type = SocketType, socket = Socket, config_db = ConfigDB} = ModData, StatusCode, PhraseArgs) -> + ?hdrd("send status", [{status_code, StatusCode}, + {phrase_args, PhraseArgs}]), + ReasonPhrase = httpd_util:reason_phrase(StatusCode), Message = httpd_util:message(StatusCode, PhraseArgs, ConfigDB), Body = get_body(ReasonPhrase, Message), - send_header(ModData, StatusCode, [{content_type, "text/html"}, - {content_length, integer_to_list(length(Body))}]), + ?hdrt("send status - header", [{reason_phrase, ReasonPhrase}, + {message, Message}]), + send_header(ModData, StatusCode, + [{content_type, "text/html"}, + {content_length, integer_to_list(length(Body))}]), + httpd_socket:deliver(SocketType, Socket, Body). @@ -345,8 +353,9 @@ transform({Field, Value}) when is_list(Field) -> %% Leave this method and go on to the newer form of response %% OTP-4408 %%---------------------------------------------------------------------- -send_response_old(#mod{method = "HEAD"} = ModData, +send_response_old(#mod{method = "HEAD"} = ModData, StatusCode, Response) -> + NewResponse = lists:flatten(Response), case httpd_util:split(NewResponse, [?CR, ?LF, ?CR, ?LF],2) of diff --git a/lib/inets/src/http_server/httpd_util.erl b/lib/inets/src/http_server/httpd_util.erl index c051422529..b0b18b9c3d 100644 --- a/lib/inets/src/http_server/httpd_util.erl +++ b/lib/inets/src/http_server/httpd_util.erl @@ -178,11 +178,12 @@ message(301,URL,_) -> "The document has moved <A HREF=\""++ maybe_encode(URL) ++"\">here</A>."; message(304, _URL,_) -> "The document has not been changed."; -message(400,none,_) -> - "Your browser sent a query that this server could not understand."; -message(400,Msg,_) -> - "Your browser sent a query that this server could not understand. "++ http_util:html_encode(Msg); -message(401,none,_) -> +message(400, none, _) -> + "Your browser sent a query that this server could not understand. "; +message(400, Msg, _) -> + "Your browser sent a query that this server could not understand. " ++ + html_encode(Msg); +message(401, none, _) -> "This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong @@ -190,40 +191,49 @@ credentials (e.g., bad password), or your browser doesn't understand how to supply the credentials required."; message(403,RequestURI,_) -> - "You don't have permission to access "++ http_util:html_encode(RequestURI) ++" on this server."; + "You don't have permission to access " ++ + html_encode(RequestURI) ++ + " on this server."; message(404,RequestURI,_) -> - "The requested URL " ++ http_util:html_encode(RequestURI) ++ " was not found on this server."; + "The requested URL " ++ + html_encode(RequestURI) ++ + " was not found on this server."; message(408, Timeout, _) -> Timeout; message(412,none,_) -> "The requested preconditions were false"; message(413, Reason,_) -> - "Entity: " ++ http_util:html_encode(Reason); + "Entity: " ++ html_encode(Reason); message(414,ReasonPhrase,_) -> - "Message "++ http_util:html_encode(ReasonPhrase) ++"."; + "Message " ++ html_encode(ReasonPhrase) ++ "."; message(416,ReasonPhrase,_) -> - http_util:html_encode(ReasonPhrase); + html_encode(ReasonPhrase); message(500,_,ConfigDB) -> ServerAdmin=lookup(ConfigDB,server_admin,"unknown@unknown"), "The server encountered an internal error or " "misconfiguration and was unable to complete " "your request.<P>Please contact the server administrator " - ++ http_util:html_encode(ServerAdmin) ++ ", and inform them of the time the error occurred " + ++ html_encode(ServerAdmin) ++ + ", and inform them of the time the error occurred " "and anything you might have done that may have caused the error."; message(501,{Method, RequestURI, HTTPVersion}, _ConfigDB) -> if is_atom(Method) -> - http_util:html_encode(atom_to_list(Method))++ - " to "++ http_util:html_encode(RequestURI)++" ("++ http_util:html_encode(HTTPVersion)++") not supported."; + atom_to_list(Method) ++ + " to " ++ + html_encode(RequestURI) ++ + " (" ++ HTTPVersion ++ ") not supported."; is_list(Method) -> - http_util:html_encode(Method)++ - " to "++ http_util:html_encode(RequestURI)++" ("++ http_util:html_encode(HTTPVersion)++") not supported." + Method ++ + " to " ++ + html_encode(RequestURI) ++ + " (" ++ HTTPVersion ++ ") not supported." end; message(503, String, _ConfigDB) -> - "This service in unavailable due to: "++ http_util:html_encode(String). + "This service in unavailable due to: " ++ html_encode(String). maybe_encode(URI) -> Decoded = try http_uri:decode(URI) of @@ -233,6 +243,15 @@ maybe_encode(URI) -> end, http_uri:encode(Decoded). +html_encode(String) -> + try http_uri:decode(String) of + Decoded when is_list(Decoded) -> + http_util:html_encode(Decoded) + catch + _:_ -> + http_util:html_encode(String) + end. + %%convert_rfc_date(Date)->{{YYYY,MM,DD},{HH,MIN,SEC}} convert_request_date([D,A,Y,DateType| Rest])-> @@ -245,7 +264,7 @@ convert_request_date([D,A,Y,DateType| Rest])-> fun convert_rfc850_date/1 end, case catch Func([D,A,Y,DateType| Rest]) of - {ok,Date} -> + {ok, Date} -> Date; _Error-> bad_date diff --git a/lib/inets/src/http_server/mod_auth_mnesia.erl b/lib/inets/src/http_server/mod_auth_mnesia.erl index b7b9520649..91beb0e062 100644 --- a/lib/inets/src/http_server/mod_auth_mnesia.erl +++ b/lib/inets/src/http_server/mod_auth_mnesia.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/inets/src/http_server/mod_responsecontrol.erl b/lib/inets/src/http_server/mod_responsecontrol.erl index 5d5b60cdbd..989f45db20 100644 --- a/lib/inets/src/http_server/mod_responsecontrol.erl +++ b/lib/inets/src/http_server/mod_responsecontrol.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2010. All Rights Reserved. +%% Copyright Ericsson AB 2001-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -209,14 +209,14 @@ compare_etags(Tag,Etags) -> nomatch end. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%%Control if the file is modificated %% -%% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% %% +%% Control if the file is modificated %% +%% %% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%---------------------------------------------------------------------- -%%Control the If-Modified-Since and If-Not-Modified-Since header fields +%% Control the If-Modified-Since and If-Not-Modified-Since header fields %%---------------------------------------------------------------------- control_modification(Path,Info,FileInfo)-> ?DEBUG("control_modification() -> entry",[]), @@ -227,6 +227,8 @@ control_modification(Path,Info,FileInfo)-> continue; unmodified-> {304, Info, Path}; + {bad_date, _} = BadDate-> + {400, Info, BadDate}; undefined -> case control_modification_data(Info, FileInfo#file_info.mtime, @@ -253,21 +255,27 @@ control_modification_data(Info, ModificationTime, HeaderField)-> undefined-> undefined; LastModified0 -> - LastModified = calendar:universal_time_to_local_time( - httpd_util:convert_request_date(LastModified0)), - ?DEBUG("control_modification_data() -> " - "~n Request-Field: ~s" - "~n FileLastModified: ~p" - "~n FieldValue: ~p", - [HeaderField, ModificationTime, LastModified]), - FileTime = - calendar:datetime_to_gregorian_seconds(ModificationTime), - FieldTime = calendar:datetime_to_gregorian_seconds(LastModified), - if - FileTime =< FieldTime -> - ?DEBUG("File unmodified~n", []), unmodified; - FileTime >= FieldTime -> - ?DEBUG("File modified~n", []), modified + case httpd_util:convert_request_date(LastModified0) of + bad_date -> + {bad_date, LastModified0}; + ConvertedReqDate -> + LastModified = + calendar:universal_time_to_local_time(ConvertedReqDate), + ?DEBUG("control_modification_data() -> " + "~n Request-Field: ~s" + "~n FileLastModified: ~p" + "~n FieldValue: ~p", + [HeaderField, ModificationTime, LastModified]), + FileTime = + calendar:datetime_to_gregorian_seconds(ModificationTime), + FieldTime = + calendar:datetime_to_gregorian_seconds(LastModified), + if + FileTime =< FieldTime -> + ?DEBUG("File unmodified~n", []), unmodified; + FileTime >= FieldTime -> + ?DEBUG("File modified~n", []), modified + end end end. @@ -285,6 +293,9 @@ strip_date([C | Rest]) -> send_return_value({412,_,_}, _FileInfo)-> {status,{412,none,"Precondition Failed"}}; +send_return_value({400,_, {bad_date, BadDate}}, _FileInfo)-> + {status, {400, none, "Bad date: " ++ BadDate}}; + send_return_value({304,Info,Path}, FileInfo)-> Suffix = httpd_util:suffix(Path), MimeType = httpd_util:lookup_mime_default(Info#mod.config_db,Suffix, diff --git a/lib/inets/src/inets_app/Makefile b/lib/inets/src/inets_app/Makefile index 20e22917e2..63ab0a5bae 100644 --- a/lib/inets/src/inets_app/Makefile +++ b/lib/inets/src/inets_app/Makefile @@ -30,6 +30,7 @@ include ../../vsn.mk VSN = $(INETS_VSN) + # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- @@ -41,8 +42,8 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # ---------------------------------------------------- MODULES = \ - inets_service \ inets \ + inets_service \ inets_app \ inets_sup \ inets_regexp diff --git a/lib/inets/src/inets_app/inets.app.src b/lib/inets/src/inets_app/inets.app.src index cb036157a5..4d0defb329 100644 --- a/lib/inets/src/inets_app/inets.app.src +++ b/lib/inets/src/inets_app/inets.app.src @@ -34,8 +34,7 @@ ftp_sup, %% HTTP client: - http, %% Old client API module - httpc, %% New client API module + httpc, httpc_handler, httpc_handler_sup, httpc_manager, diff --git a/lib/inets/src/inets_app/inets.appup.src b/lib/inets/src/inets_app/inets.appup.src index 8b0fcb185d..779dd8e439 100644 --- a/lib/inets/src/inets_app/inets.appup.src +++ b/lib/inets/src/inets_app/inets.appup.src @@ -18,65 +18,47 @@ {"%VSN%", [ - {"5.6", - [ - {load_module, httpc, soft_purge, soft_purge, [httpc_manager]}, - {load_module, http_transport, soft_purge, soft_purge, [http_transport]}, - {update, httpc_handler, soft, soft_purge, soft_purge, []}, - {update, httpc_manager, soft, soft_purge, soft_purge, [httpc_handler]}, - {update, ftp, soft, soft_purge, soft_purge, []} - ] - }, - {"5.5.2", + {"5.7.2", [ {restart_application, inets} ] }, - {"5.5.1", + {"5.7.1", [ {restart_application, inets} ] }, - {"5.5", + {"5.7", [ {restart_application, inets} ] }, - {"5.4", + {"5.6", [ {restart_application, inets} ] } ], [ - {"5.6", - [ - {load_module, httpc, soft_purge, soft_purge, [httpc_manager]}, - {load_module, http_transport, soft_purge, soft_purge, [http_transport]}, - {update, httpc_handler, soft, soft_purge, soft_purge, []}, - {update, httpc_manager, soft, soft_purge, soft_purge, [httpc_handler]}, - {update, ftp, soft, soft_purge, soft_purge, []} - ] - }, - {"5.5.2", + {"5.7.2", [ {restart_application, inets} ] - }, - {"5.5.1", + }, + {"5.7.1", [ {restart_application, inets} ] }, - {"5.5", + {"5.7", [ {restart_application, inets} ] - }, - {"5.4", + }, + {"5.6", [ {restart_application, inets} ] - } + } ] }. diff --git a/lib/inets/src/inets_app/inets.mk b/lib/inets/src/inets_app/inets.mk index b6e9fe1d96..35fb0d7eca 100644 --- a/lib/inets/src/inets_app/inets.mk +++ b/lib/inets/src/inets_app/inets.mk @@ -33,6 +33,10 @@ ifeq ($(WARN_UNUSED_WARS), true) ERL_COMPILE_FLAGS += +warn_unused_vars endif +ifeq ($(shell erl -noshell -eval 'io:format("~4s", [erlang:system_info(otp_release)])' -s init stop), R14B) +INETS_ERL_COMPILE_FLAGS += -D'OTP-R14B-COMPILER' +endif + INETS_APP_VSN_COMPILE_FLAGS = \ +'{parse_transform,sys_pre_attributes}' \ +'{attribute,insert,app_vsn,$(APP_VSN)}' diff --git a/lib/inets/src/inets_app/inets_service.erl b/lib/inets/src/inets_app/inets_service.erl index e9eb9892f2..a057a51e2c 100644 --- a/lib/inets/src/inets_app/inets_service.erl +++ b/lib/inets/src/inets_app/inets_service.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,24 +20,35 @@ -module(inets_service). +-ifdef('OTP-R14B-COMPILER'). + -export([behaviour_info/1]). behaviour_info(callbacks) -> - [{start_standalone, 1}, - {start_service, 1}, - {stop_service, 1}, - {services, 0}, - {service_info, 1}]; + [{start_standalone, 1}, + {start_service, 1}, + {stop_service, 1}, + {services, 0}, + {service_info, 1}]; behaviour_info(_) -> - undefined. + undefined. + +-else. %% Starts service stand-alone %% start_standalone(Config) -> % {ok, Pid} | {error, Reason} %% <service>:start_link(Config). +-callback start_standalone(Config :: term()) -> + {ok, pid()} | {error, Reason :: term()}. + %% Starts service as part of inets %% start_service(Config) -> % {ok, Pid} | {error, Reason} %% <service_sup>:start_child(Config). + +-callback start_service(Config :: term()) -> + {ok, pid()} | {error, Reason :: term()}. + %% Stop service %% stop_service(Pid) -> % ok | {error, Reason} %% <service_sup>:stop_child(maybe_map_pid_to_other_ref(Pid)). @@ -51,6 +62,9 @@ behaviour_info(_) -> %% Error %% end. +-callback stop_service(Service :: term()) -> + ok | {error, Reason :: term()}. + %% Returns list of running services. Services started as stand alone %% are not listed %% services() -> % [{Service, Pid}] @@ -59,7 +73,14 @@ behaviour_info(_) -> %% [{httpc, Pid} || {_, Pid, _, _} <- %% supervisor:which_children(httpc_profile_sup)]. +-callback services() -> + [{Service :: term(), pid()}]. -%% service_info() -> [{Property, Value}] | {error, Reason} +%% service_info() -> {ok, [{Property, Value}]} | {error, Reason} %% ex: httpc:service_info() -> [{profile, ProfileName}] %% httpd:service_info() -> [{host, Host}, {port, Port}] + +-callback service_info(Service :: term()) -> + {ok, [{Property :: term(), Value :: term()}]} | {error, Reason :: term()}. + +-endif. diff --git a/lib/inets/src/tftp/tftp.erl b/lib/inets/src/tftp/tftp.erl index bfdb4c0030..8d8fce388d 100644 --- a/lib/inets/src/tftp/tftp.erl +++ b/lib/inets/src/tftp/tftp.erl @@ -215,8 +215,6 @@ start/0 ]). --export([behaviour_info/1]). - %% Application local functions -export([ start_standalone/1, @@ -226,14 +224,67 @@ service_info/1 ]). +-ifdef('OTP-R14B-COMPILER'). + +-export([behaviour_info/1]). behaviour_info(callbacks) -> - [{prepare, 6}, {open, 6}, {read, 1}, {write, 2}, {abort, 3}]; + [{prepare, 6}, + {open, 6}, + {read, 1}, + {write, 2}, + {abort, 3}]; behaviour_info(_) -> undefined. +-else. + +-type peer() :: {PeerType :: inet | inet6, + PeerHost :: inet:ip_address(), + PeerPort :: port()}. + +-type access() :: read | write. + +-type options() :: [{Key :: string(), Value :: string()}]. + +-type error_code() :: undef | enoent | eacces | enospc | + badop | eexist | baduser | badopt | + integer(). + +-callback prepare(Peer :: peer(), + Access :: access(), + Filename :: file:name(), + Mode :: string(), + SuggestedOptions :: options(), + InitialState :: [] | [{root_dir, string()}]) -> + {ok, AcceptedOptions :: options(), NewState :: term()} | + {error, {Code :: error_code(), string()}}. + +-callback open(Peer :: peer(), + Access :: access(), + Filename :: file:name(), + Mode :: string(), + SuggestedOptions :: options(), + State :: [] | [{root_dir, string()}] | term()) -> + {ok, AcceptedOptions :: options(), NewState :: term()} | + {error, {Code :: error_code(), string()}}. + +-callback read(State :: term()) -> {more, binary(), NewState :: term()} | + {last, binary(), integer()} | + {error, {Code :: error_code(), string()}}. + +-callback write(binary(), State :: term()) -> + {more, NewState :: term()} | + {last, FileSize :: integer()} | + {error, {Code :: error_code(), string()}}. + +-callback abort(Code :: error_code(), string(), State :: term()) -> 'ok'. + +-endif. + -include("tftp.hrl"). + %%------------------------------------------------------------------- %% read_file(RemoteFilename, LocalFilename, Options) -> %% {ok, LastCallbackState} | {error, Reason} diff --git a/lib/inets/test/ftp_suite_lib.erl b/lib/inets/test/ftp_suite_lib.erl index 3ebd02229e..ffb58c91b6 100644 --- a/lib/inets/test/ftp_suite_lib.erl +++ b/lib/inets/test/ftp_suite_lib.erl @@ -196,7 +196,9 @@ test_filenames() -> %% variable, but should NOT alter/remove any existing entries. %%-------------------------------------------------------------------- init_per_testcase(Case, Config) - when (Case =:= open) orelse (Case =:= open_port) -> + when (Case =:= open) orelse + (Case =:= open_port) -> + put(ftp_testcase, Case), io:format(user, "~n~n*** INIT ~w:~w ***~n~n", [?MODULE, Case]), inets:start(), NewConfig = data_dir(Config), @@ -266,7 +268,7 @@ do_init_per_testcase(Case, Config) -> end, Opts2 = case string:tokens(atom_to_list(Case), [$_]) of - [_, "active" | _] -> + ["active" | _] -> [{mode, active} | Opts1]; _ -> [{mode, passive} | Opts1] @@ -367,8 +369,11 @@ open(Config) when is_list(Config) -> tc_open(Host) -> + p("tc_open -> entry with" + "~n Host: ~p", [Host]), {ok, Pid} = ?ftp_open(Host, []), ok = ftp:close(Pid), + p("tc_open -> try (ok) open 1"), {ok, Pid1} = ftp:open({option_list, [{host,Host}, {port, ?FTP_PORT}, @@ -376,11 +381,13 @@ tc_open(Host) -> {timeout, 30000}]}), ok = ftp:close(Pid1), + p("tc_open -> try (fail) open 2"), {error, ehost} = ftp:open({option_list, [{port, ?FTP_PORT}, {flags, [verbose]}]}), {ok, Pid2} = ftp:open(Host), ok = ftp:close(Pid2), + p("tc_open -> try (ok) open 3"), {ok, NewHost} = inet:getaddr(Host, inet), {ok, Pid3} = ftp:open(NewHost), ftp:user(Pid3, ?FTP_USER, ?FTP_PASS), @@ -392,33 +399,68 @@ tc_open(Host) -> %% Bad input that has default values are ignored and the defult %% is used. + p("tc_open -> try (ok) open 4"), {ok, Pid4} = - ftp:open({option_list, [{host, Host}, {port, badarg}, - {flags, [verbose]}, + ftp:open({option_list, [{host, Host}, + {port, badarg}, + {flags, [verbose]}, {timeout, 30000}]}), test_server:sleep(100), ok = ftp:close(Pid4), + + p("tc_open -> try (ok) open 5"), {ok, Pid5} = - ftp:open({option_list, [{host, Host}, {port, ?FTP_PORT}, - {flags, [verbose]}, + ftp:open({option_list, [{host, Host}, + {port, ?FTP_PORT}, + {flags, [verbose]}, {timeout, -42}]}), test_server:sleep(100), ok = ftp:close(Pid5), + + p("tc_open -> try (ok) open 6"), {ok, Pid6} = - ftp:open({option_list, [{host, Host}, {port, ?FTP_PORT}, + ftp:open({option_list, [{host, Host}, + {port, ?FTP_PORT}, {flags, [verbose]}, - {mode, cool}]}), + {mode, cool}]}), test_server:sleep(100), ok = ftp:close(Pid6), + p("tc_open -> try (ok) open 7"), {ok, Pid7} = ftp:open(Host, [{port, ?FTP_PORT}, {verbose, true}, {timeout, 30000}]), ok = ftp:close(Pid7), + p("tc_open -> try (ok) open 8"), {ok, Pid8} = ftp:open(Host, ?FTP_PORT), ok = ftp:close(Pid8), + p("tc_open -> try (ok) open 9"), + {ok, Pid9} = + ftp:open(Host, [{port, ?FTP_PORT}, + {verbose, true}, + {timeout, 30000}, + {dtimeout, -99}]), + ok = ftp:close(Pid9), + + p("tc_open -> try (ok) open 10"), + {ok, Pid10} = + ftp:open(Host, [{port, ?FTP_PORT}, + {verbose, true}, + {timeout, 30000}, + {dtimeout, "foobar"}]), + ok = ftp:close(Pid10), + + p("tc_open -> try (ok) open 11"), + {ok, Pid11} = + ftp:open(Host, [{port, ?FTP_PORT}, + {verbose, true}, + {timeout, 30000}, + {dtimeout, 1}]), + ok = ftp:close(Pid11), + + p("tc_open -> done"), ok. @@ -445,7 +487,7 @@ passive_user(suite) -> []; passive_user(Config) when is_list(Config) -> Pid = ?config(ftp, Config), - io:format("Pid: ~p~n",[Pid]), + p("Pid: ~p",[Pid]), do_user(Pid). @@ -967,13 +1009,13 @@ api_missuse(doc)-> ["Test that behaviour of the ftp process if the api is abused"]; api_missuse(suite) -> []; api_missuse(Config) when is_list(Config) -> - io:format("api_missuse -> entry~n", []), + p("api_missuse -> entry"), Flag = process_flag(trap_exit, true), Pid = ?config(ftp, Config), Host = ftp_host(Config), %% Serious programming fault, connetion will be shut down - io:format("api_missuse -> verify bad call termination (~p)~n", [Pid]), + p("api_missuse -> verify bad call termination (~p)", [Pid]), case (catch gen_server:call(Pid, {self(), foobar, 10}, infinity)) of {error, {connection_terminated, 'API_violation'}} -> ok; @@ -983,23 +1025,23 @@ api_missuse(Config) when is_list(Config) -> test_server:sleep(500), undefined = process_info(Pid, status), - io:format("api_missuse -> start new client~n", []), + p("api_missuse -> start new client"), {ok, Pid2} = ?ftp_open(Host, []), %% Serious programming fault, connetion will be shut down - io:format("api_missuse -> verify bad cast termination~n", []), + p("api_missuse -> verify bad cast termination"), gen_server:cast(Pid2, {self(), foobar, 10}), test_server:sleep(500), undefined = process_info(Pid2, status), - io:format("api_missuse -> start new client~n", []), + p("api_missuse -> start new client"), {ok, Pid3} = ?ftp_open(Host, []), %% Could be an innocent misstake the connection lives. - io:format("api_missuse -> verify bad bang~n", []), + p("api_missuse -> verify bad bang"), Pid3 ! foobar, test_server:sleep(500), {status, _} = process_info(Pid3, status), process_flag(trap_exit, Flag), - io:format("api_missuse -> done~n", []), + p("api_missuse -> done"), ok. @@ -1567,9 +1609,9 @@ split([], I, Is) -> lists:reverse([lists:reverse(I)| Is]). do_ftp_open(Host, Opts) -> - io:format("do_ftp_open -> entry with" - "~n Host: ~p" - "~n Opts: ~p", [Host, Opts]), + p("do_ftp_open -> entry with" + "~n Host: ~p" + "~n Opts: ~p", [Host, Opts]), case ftp:open(Host, Opts) of {ok, _} = OK -> OK; @@ -1595,7 +1637,7 @@ passwd() -> ftpd_hosts(Config) -> DataDir = ?config(data_dir, Config), FileName = filename:join([DataDir, "../ftp_SUITE_data/", ftpd_hosts]), - io:format("FileName: ~p~n", [FileName]), + p("FileName: ~p", [FileName]), case file:consult(FileName) of {ok, [Hosts]} when is_list(Hosts) -> Hosts; diff --git a/lib/inets/test/ftp_windows_2003_server_test.erl b/lib/inets/test/ftp_windows_2003_server_test.erl index 57f1ae8358..32f25713f8 100644 --- a/lib/inets/test/ftp_windows_2003_server_test.erl +++ b/lib/inets/test/ftp_windows_2003_server_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -87,14 +87,22 @@ end_per_testcase(Case, Config) -> %% Description: Returns a list of all test cases in this test suite %%-------------------------------------------------------------------- all() -> - [open, open_port, {group, passive}, {group, active}, - api_missuse, not_owner, {group, progress_report}]. + [ + open, + open_port, + {group, passive}, + {group, active}, + api_missuse, + not_owner, + {group, progress_report} + ]. groups() -> - [{passive, [], ftp_suite_lib:passive(suite)}, - {active, [], ftp_suite_lib:active(suite)}, - {progress_report, [], - ftp_suite_lib:progress_report(suite)}]. + [ + {passive, [], ftp_suite_lib:passive(suite)}, + {active, [], ftp_suite_lib:active(suite)}, + {progress_report, [], ftp_suite_lib:progress_report(suite)} + ]. init_per_group(_GroupName, Config) -> Config. diff --git a/lib/inets/test/http_format_SUITE.erl b/lib/inets/test/http_format_SUITE.erl index 931ac6e024..04c7358715 100644 --- a/lib/inets/test/http_format_SUITE.erl +++ b/lib/inets/test/http_format_SUITE.erl @@ -584,7 +584,9 @@ convert_netscapecookie_date(Config) when is_list(Config) -> http_util:convert_netscapecookie_date("Sun, 12-Dec-06 08:59:38 GMT"), {{2006,12,12},{8,59,38}} = http_util:convert_netscapecookie_date("Sun 12-Dec-06 08:59:38 GMT"), - ok. + {{2036,1,1},{8,0,1}} = + http_util:convert_netscapecookie_date("Tue Jan 01 08:00:01 2036 GMT"), + ok. %%-------------------------------------------------------------------- %%% Internal functions diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 6edd5371af..d86e52f3d5 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -113,13 +113,10 @@ groups() -> proxy_page_does_not_exist, proxy_https_not_supported]}, {ssl, [], [ssl_head, - ossl_head, essl_head, ssl_get, - ossl_get, essl_get, ssl_trace, - ossl_trace, essl_trace]}, {stream, [], [http_stream, http_stream_once, @@ -253,10 +250,10 @@ init_per_testcase(Case, Config) -> init_per_testcase(Case, 2, Config). init_per_testcase(Case, Timeout, Config) -> - io:format(user, "~n~n*** INIT ~w:~w[~w] ***~n~n", - [?MODULE, Case, Timeout]), - PrivDir = ?config(priv_dir, Config), - tsp("init_per_testcase -> stop inets"), + io:format(user, + "~n~n*** INIT ~w:~w[~w] ***" + "~n~n", [?MODULE, Case, Timeout]), + PrivDir = ?config(priv_dir, Config), application:stop(inets), Dog = test_server:timetrap(inets_test_lib:minutes(Timeout)), TmpConfig = lists:keydelete(watchdog, 1, Config), @@ -273,10 +270,6 @@ init_per_testcase(Case, Timeout, Config) -> init_per_testcase_ssl(ssl, PrivDir, SslConfFile, [{watchdog, Dog} | TmpConfig]); - [$o, $s, $s, $l | _] -> - init_per_testcase_ssl(ossl, PrivDir, SslConfFile, - [{watchdog, Dog} | TmpConfig]); - [$e, $s, $s, $l | _] -> init_per_testcase_ssl(essl, PrivDir, SslConfFile, [{watchdog, Dog} | TmpConfig]); @@ -296,12 +289,12 @@ init_per_testcase(Case, Timeout, Config) -> throw:{error, {failed_starting, App, _}} -> SkipString = "Could not start " ++ atom_to_list(App), - {skip, SkipString}; - _:X -> + skip(SkipString); + _:X -> SkipString = lists:flatten( io_lib:format("Failed starting apps: ~p", [X])), - {skip, SkipString} + skip(SkipString) end; _ -> @@ -330,14 +323,14 @@ init_per_testcase(Case, Timeout, Config) -> ], case lists:member(Rest, BadCases) of true -> - [{skip, "TC and server not compatible"}| + [skip("TC and server not compatible") | TmpConfig]; false -> inets:start(), [{watchdog, Dog} | TmpConfig] end; false -> - [{skip, "proxy not responding"} | TmpConfig] + [skip("proxy not responding") | TmpConfig] end end; @@ -367,20 +360,19 @@ init_per_testcase(Case, Timeout, Config) -> io_lib:format("Failed starting apps: ~p", [X])), {skip, SkipString} end; + _ -> TmpConfig2 = lists:keydelete(local_server, 1, TmpConfig), - Server = - %% Will start inets - inets_test_lib:start_http_server( - filename:join(PrivDir, IpConfFile)), + %% Will start inets + Server = start_http_server(PrivDir, IpConfFile), [{watchdog, Dog}, {local_server, Server} | TmpConfig2] end, %% This will fail for the ipv6_ - cases (but that is ok) - httpc:set_options([{proxy, {{?PROXY, ?PROXY_PORT}, - ["localhost", ?IPV6_LOCAL_HOST]}}, - {ipfamily, inet6fb4}]), - + ProxyExceptions = ["localhost", ?IPV6_LOCAL_HOST], + http:set_options([{proxy, {{?PROXY, ?PROXY_PORT}, ProxyExceptions}}]), + inets:enable_trace(max, io, httpc), + %% inets:enable_trace(max, io, all), %% snmp:set_trace([gen_tcp]), NewConfig. @@ -397,7 +389,10 @@ init_per_testcase_ssl(Tag, PrivDir, SslConfFile, Config) -> tsp("init_per_testcase(~w) -> Server: ~p", [Tag, Server]), [{local_ssl_server, Server} | Config2]. - +start_http_server(ConfDir, ConfFile) -> + inets_test_lib:start_http_server( filename:join(ConfDir, ConfFile) ). + + %%-------------------------------------------------------------------- %% Function: end_per_testcase(Case, Config) -> _ %% Case - atom() @@ -733,7 +728,7 @@ test_pipeline(URL) -> p("test_pipeline -> received reply for (async) request 2"), ok; {http, Msg1} -> - test_server:fail(Msg1) + tsf(Msg1) end; {http, {RequestId2, {{_, 200, _}, _, _}}} -> io:format("test_pipeline -> received reply for (async) request 2 - now wait for 1"), @@ -742,14 +737,14 @@ test_pipeline(URL) -> io:format("test_pipeline -> received reply for (async) request 1"), ok; {http, Msg2} -> - test_server:fail(Msg2) + tsf(Msg2) end; {http, Msg3} -> - test_server:fail(Msg3) + tsf(Msg3) after 60000 -> receive Any1 -> tsp("received crap after timeout: ~n ~p", [Any1]), - test_server:fail({error, {timeout, Any1}}) + tsf({error, {timeout, Any1}}) end end, @@ -774,7 +769,7 @@ test_pipeline(URL) -> p("test_pipeline -> expect *no* reply for cancelled (async) request 4 (for 3 secs)"), receive {http, {RequestId3, _}} -> - test_server:fail(http_cancel_request_failed) + tsf(http_cancel_request_failed) after 3000 -> ok end, @@ -787,11 +782,11 @@ test_pipeline(URL) -> tsp("Receive : ~p", [Res]), BinBody4; {http, Msg4} -> - test_server:fail(Msg4) + tsf(Msg4) after 60000 -> receive Any2 -> tsp("received crap after timeout: ~n ~p", [Any2]), - test_server:fail({error, {timeout, Any2}}) + tsf({error, {timeout, Any2}}) end end, @@ -801,7 +796,7 @@ test_pipeline(URL) -> p("test_pipeline -> ensure no unexpected incomming"), receive {http, Any} -> - test_server:fail({unexpected_message, Any}) + tsf({unexpected_message, Any}) after 500 -> ok end, @@ -823,11 +818,11 @@ http_trace(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], "TRACE /dummy.html" ++ _}} -> ok; {ok, {{_,200,_}, [_ | _], WrongBody}} -> - test_server:fail({wrong_body, WrongBody}); + tsf({wrong_body, WrongBody}); {ok, WrongReply} -> - test_server:fail({wrong_reply, WrongReply}); + tsf({wrong_reply, WrongReply}); Error -> - test_server:fail({failed, Error}) + tsf({failed, Error}) end; _ -> {skip, "Failed to start local http-server"} @@ -850,7 +845,7 @@ http_async(Config) when is_list(Config) -> {http, {RequestId, {{_, 200, _}, _, BinBody}}} -> BinBody; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, inets_test_lib:check_body(binary_to_list(Body)), @@ -860,7 +855,7 @@ http_async(Config) when is_list(Config) -> ok = httpc:cancel_request(NewRequestId), receive {http, {NewRequestId, _NewResult}} -> - test_server:fail(http_cancel_request_failed) + tsf(http_cancel_request_failed) after 3000 -> ok end; @@ -909,7 +904,7 @@ http_save_to_file_async(Config) when is_list(Config) -> {http, {RequestId, saved_to_file}} -> ok; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, {ok, Bin} = file:read_file(FilePath), @@ -1076,13 +1071,6 @@ ssl_head(suite) -> ssl_head(Config) when is_list(Config) -> ssl_head(ssl, Config). -ossl_head(doc) -> - ["Same as http_head/1 but over ssl sockets."]; -ossl_head(suite) -> - []; -ossl_head(Config) when is_list(Config) -> - ssl_head(ossl, Config). - essl_head(doc) -> ["Same as http_head/1 but over ssl sockets."]; essl_head(suite) -> @@ -1105,8 +1093,6 @@ ssl_head(SslTag, Config) -> case SslTag of ssl -> SSLOptions; - ossl -> - {ossl, SSLOptions}; essl -> {essl, SSLOptions} end, @@ -1131,13 +1117,6 @@ ssl_get(suite) -> ssl_get(Config) when is_list(Config) -> ssl_get(ssl, Config). -ossl_get(doc) -> - ["Same as http_get/1 but over ssl sockets."]; -ossl_get(suite) -> - []; -ossl_get(Config) when is_list(Config) -> - ssl_get(ossl, Config). - essl_get(doc) -> ["Same as http_get/1 but over ssl sockets."]; essl_get(suite) -> @@ -1157,8 +1136,6 @@ ssl_get(SslTag, Config) when is_list(Config) -> case SslTag of ssl -> SSLOptions; - ossl -> - {ossl, SSLOptions}; essl -> {essl, SSLOptions} end, @@ -1184,13 +1161,6 @@ ssl_trace(suite) -> ssl_trace(Config) when is_list(Config) -> ssl_trace(ssl, Config). -ossl_trace(doc) -> - ["Same as http_trace/1 but over ssl sockets."]; -ossl_trace(suite) -> - []; -ossl_trace(Config) when is_list(Config) -> - ssl_trace(ossl, Config). - essl_trace(doc) -> ["Same as http_trace/1 but over ssl sockets."]; essl_trace(suite) -> @@ -1210,8 +1180,6 @@ ssl_trace(SslTag, Config) when is_list(Config) -> case SslTag of ssl -> SSLOptions; - ossl -> - {ossl, SSLOptions}; essl -> {essl, SSLOptions} end, @@ -1482,10 +1450,10 @@ proxy_options(Config) when is_list(Config) -> {value, {"allow", _}} -> ok; _ -> - test_server:fail(http_options_request_failed) + tsf(http_options_request_failed) end; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1506,7 +1474,7 @@ proxy_head(Config) when is_list(Config) -> {ok, {{_,200, _}, [_ | _], []}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1525,7 +1493,7 @@ proxy_get(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], Body = [_ | _]}} -> inets_test_lib:check_body(Body); Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1604,7 +1572,7 @@ proxy_post(Config) when is_list(Config) -> {ok, {{_,405,_}, [_ | _], [_ | _]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1629,7 +1597,7 @@ proxy_put(Config) when is_list(Config) -> {ok, {{_,405,_}, [_ | _], [_ | _]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1654,7 +1622,7 @@ proxy_delete(Config) when is_list(Config) -> {ok, {{_,404,_}, [_ | _], [_ | _]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1710,7 +1678,7 @@ proxy_auth(Config) when is_list(Config) -> {ok, {{_,200, _}, [_ | _], [_|_]}} -> ok; Unexpected -> - test_server:fail({unexpected_result, Unexpected}) + tsf({unexpected_result, Unexpected}) end; Reason -> {skip, Reason} @@ -1796,7 +1764,7 @@ http_stream(Config) when is_list(Config) -> {http, {RequestId, stream_start, _Headers}} -> ok; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, StreamedBody = receive_streamed_body(RequestId, <<>>), @@ -1851,7 +1819,7 @@ once(URL) -> [RequestId, Pid]), Pid; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, tsp("once -> request handler: ~p", [NewPid]), @@ -1894,7 +1862,7 @@ proxy_stream(Config) when is_list(Config) -> {http, {RequestId, stream_start, _Headers}} -> ok; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end, StreamedBody = receive_streamed_body(RequestId, <<>>), @@ -1912,22 +1880,31 @@ parse_url(suite) -> []; parse_url(Config) when is_list(Config) -> %% ipv6 - {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]} - = http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html"), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html"), + {ok, {http,[],"[2010:836B:4179::836B:4179]",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{ipv6_host_with_brackets, true}]), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{ipv6_host_with_brackets, false}]), + {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = + http_uri:parse("http://[2010:836B:4179::836B:4179]/foobar.html", + [{foo, false}]), {error, {malformed_url,"http://2010:836B:4179::836B:4179/foobar.html"}} = http_uri:parse("http://2010:836B:4179::836B:4179/foobar.html"), %% ipv4 - {http,[],"127.0.0.1",80,"/foobar.html",[]} = + {ok, {http,[],"127.0.0.1",80,"/foobar.html",[]}} = http_uri:parse("http://127.0.0.1/foobar.html"), %% host - {http,[],"localhost",8888,"/foobar.html",[]} = + {ok, {http,[],"localhost",8888,"/foobar.html",[]}} = http_uri:parse("http://localhost:8888/foobar.html"), %% Userinfo - {http,"nisse:foobar","localhost",8888,"/foobar.html",[]} = + {ok, {http,"nisse:foobar","localhost",8888,"/foobar.html",[]}} = http_uri:parse("http://nisse:foobar@localhost:8888/foobar.html"), %% Scheme error @@ -1936,18 +1913,20 @@ parse_url(Config) when is_list(Config) -> http_uri:parse("localhost:8888/foobar.html"), %% Query - {http,[],"localhost",8888,"/foobar.html","?foo=bar&foobar=42"} = + {ok, {http,[],"localhost",8888,"/foobar.html","?foo=bar&foobar=42"}} = http_uri:parse("http://localhost:8888/foobar.html?foo=bar&foobar=42"), %% Esc chars - {http,[],"www.somedomain.com",80,"/%2Eabc",[]} = + {ok, {http,[],"www.somedomain.com",80,"/%2Eabc",[]}} = http_uri:parse("http://www.somedomain.com/%2Eabc"), - {http,[],"www.somedomain.com",80,"/%252Eabc",[]} = + {ok, {http,[],"www.somedomain.com",80,"/%252Eabc",[]}} = http_uri:parse("http://www.somedomain.com/%252Eabc"), - {http,[],"www.somedomain.com",80,"/%25abc",[]} = + {ok, {http,[],"www.somedomain.com",80,"/%25abc",[]}} = http_uri:parse("http://www.somedomain.com/%25abc"), - {http,[],"www.somedomain.com",80,"/%25abc", "?foo=bar"} = + {ok, {http,[],"www.somedomain.com",80,"/%25abc", "?foo=bar"}} = http_uri:parse("http://www.somedomain.com/%25abc?foo=bar"), + + ok. @@ -2092,12 +2071,14 @@ http_invalid_http(Config) when is_list(Config) -> %%------------------------------------------------------------------------- +-define(GOOGLE, "www.google.com"). + hexed_query_otp_6191(doc) -> []; hexed_query_otp_6191(suite) -> []; hexed_query_otp_6191(Config) when is_list(Config) -> - Google = "www.google.com", + Google = ?GOOGLE, GoogleSearch = "http://" ++ Google ++ "/search", Search1 = "?hl=en&q=a%D1%85%D1%83%D0%B9&btnG=Google+Search", URI1 = GoogleSearch ++ Search1, @@ -2106,11 +2087,32 @@ hexed_query_otp_6191(Config) when is_list(Config) -> Search3 = "?hl=en&q=%foo", URI3 = GoogleSearch ++ Search3, - {http, [], Google, 80, "/search", _} = http_uri:parse(URI1), - {http, [], Google, 80, "/search", _} = http_uri:parse(URI2), - {http, [], Google, 80, "/search", _} = http_uri:parse(URI3), + Verify1 = + fun({http, [], ?GOOGLE, 80, "/search", _}) -> ok; + (_) -> error + end, + Verify2 = Verify1, + Verify3 = Verify1, + verify_uri(URI1, Verify1), + verify_uri(URI2, Verify2), + verify_uri(URI3, Verify3), ok. +verify_uri(URI, Verify) -> + case http_uri:parse(URI) of + {ok, ParsedURI} -> + case Verify(ParsedURI) of + ok -> + ok; + error -> + Reason = {unexpected_parse_result, URI, ParsedURI}, + ERROR = {error, Reason}, + throw(ERROR) + end; + {error, _} = ERROR -> + throw(ERROR) + end. + %%------------------------------------------------------------------------- @@ -2979,7 +2981,7 @@ receive_streamed_body(RequestId, Body) -> {http, {RequestId, stream_end, _Headers}} -> Body; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end. receive_streamed_body(RequestId, Body, Pid) -> @@ -2993,7 +2995,7 @@ receive_streamed_body(RequestId, Body, Pid) -> {http, {RequestId, stream_end, _Headers}} -> Body; {http, Msg} -> - test_server:fail(Msg) + tsf(Msg) end. %% Perform a synchronous stop @@ -3038,10 +3040,6 @@ dummy_server_init(Caller, essl, IpV, SSLOptions) -> BaseOpts = [{ssl_imp, new}, {backlog, 128}, binary, {reuseaddr,true}, {active, false} | SSLOptions], - dummy_ssl_server_init(Caller, BaseOpts, IpV); -dummy_server_init(Caller, ossl, IpV, SSLOptions) -> - BaseOpts = [{ssl_imp, old}, - {backlog, 128}, binary, {active, false} | SSLOptions], dummy_ssl_server_init(Caller, BaseOpts, IpV). dummy_ssl_server_init(Caller, BaseOpts, IpV) -> @@ -3455,7 +3453,7 @@ handle_auth("Basic " ++ UserInfo, Challange, DefaultResponse) -> end. check_cookie([]) -> - test_server:fail(no_cookie_header); + tsf(no_cookie_header); check_cookie(["cookie:" ++ _Value | _]) -> ok; check_cookie([_Head | Tail]) -> @@ -3513,9 +3511,9 @@ p(F, A) -> io:format("~p ~w:" ++ F ++ "~n", [self(), ?MODULE | A]). tsp(F) -> - tsp(F, []). + inets_test_lib:tsp(F). tsp(F, A) -> - test_server:format("~p ~p:" ++ F ++ "~n", [self(), ?MODULE | A]). + inets_test_lib:tsp(F, A). tsf(Reason) -> test_server:fail(Reason). @@ -3570,3 +3568,6 @@ ensure_started(App) when is_atom(App) -> throw({error, {failed_starting, App, Error}}) end. + +skip(Reason) -> + {skip, Reason}. diff --git a/lib/inets/test/httpc_cookie_SUITE.erl b/lib/inets/test/httpc_cookie_SUITE.erl index feef5f1eea..93dbc270c5 100644 --- a/lib/inets/test/httpc_cookie_SUITE.erl +++ b/lib/inets/test/httpc_cookie_SUITE.erl @@ -55,7 +55,7 @@ init_per_testcase(session_cookies_only = Case, Config0) -> "~n Config0: ~p", [Case, Config0]), Config = init_workdir(Case, Config0), application:start(inets), - http:set_options([{cookies, verify}]), + httpc:set_options([{cookies, verify}]), watch_dog(Config); init_per_testcase(Case, Config0) -> @@ -66,7 +66,7 @@ init_per_testcase(Case, Config0) -> application:load(inets), application:set_env(inets, services, [{httpc, {default, CaseDir}}]), application:start(inets), - http:set_options([{cookies, verify}]), + httpc:set_options([{cookies, verify}]), watch_dog(Config). watch_dog(Config) -> @@ -119,10 +119,18 @@ end_per_testcase(Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [session_cookies_only, netscape_cookies, cookie_cancel, - cookie_expires, persistent_cookie, domain_cookie, - secure_cookie, update_cookie, update_cookie_session, - cookie_attributes]. + [ + session_cookies_only, + netscape_cookies, + cookie_cancel, + cookie_expires, + persistent_cookie, + domain_cookie, + secure_cookie, + update_cookie, + update_cookie_session, + cookie_attributes + ]. groups() -> []. @@ -152,12 +160,12 @@ session_cookies_only(Config) when is_list(Config) -> SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/;" ";max-age=60000"}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie","$Version=0; test_cookie=true; $Path=/"} - = http:cookie_header(?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), + {"cookie", "$Version=0; test_cookie=true; $Path=/"} = + httpc:cookie_header(?URL), application:stop(inets), application:start(inets), - {"cookie",""} = http:cookie_header(?URL), + {"cookie", ""} = httpc:cookie_header(?URL), tsp("session_cookies_only -> Cookies 2: ~p", [httpc:which_cookies()]), ok. @@ -172,9 +180,9 @@ netscape_cookies(Config) when is_list(Config) -> Expires = future_netscape_date(), SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/; " "expires=" ++ Expires}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie","$Version=0; test_cookie=true; $Path=/"} = - http:cookie_header(?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), + {"cookie", "$Version=0; test_cookie=true; $Path=/"} = + httpc:cookie_header(?URL), tsp("netscape_cookies -> Cookies 2: ~p", [httpc:which_cookies()]), ok. @@ -189,13 +197,13 @@ cookie_cancel(Config) when is_list(Config) -> SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/;" "max-age=60000"}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie","$Version=0; test_cookie=true; $Path=/"} - = http:cookie_header(?URL), - NewSetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/;" - "max-age=0"}], - http:verify_cookies(NewSetCookieHeaders, ?URL), - {"cookie", ""} = http:cookie_header(?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), + {"cookie", "$Version=0; test_cookie=true; $Path=/"} = + httpc:cookie_header(?URL), + NewSetCookieHeaders = + [{"set-cookie", "test_cookie=true; path=/;max-age=0"}], + httpc:store_cookies(NewSetCookieHeaders, ?URL), + {"cookie", ""} = httpc:cookie_header(?URL), tsp("cookie_cancel -> Cookies 2: ~p", [httpc:which_cookies()]), ok. @@ -209,11 +217,11 @@ cookie_expires(Config) when is_list(Config) -> SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/;" "max-age=5"}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie","$Version=0; test_cookie=true; $Path=/"} - = http:cookie_header(?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), + {"cookie", "$Version=0; test_cookie=true; $Path=/"} = + httpc:cookie_header(?URL), test_server:sleep(10000), - {"cookie", ""} = http:cookie_header(?URL), + {"cookie", ""} = httpc:cookie_header(?URL), tsp("cookie_expires -> Cookies 2: ~p", [httpc:which_cookies()]), ok. @@ -227,16 +235,16 @@ persistent_cookie(Config) when is_list(Config)-> SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/;" "max-age=60000"}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie","$Version=0; test_cookie=true; $Path=/"} = - http:cookie_header(?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), + {"cookie", "$Version=0; test_cookie=true; $Path=/"} = + httpc:cookie_header(?URL), CaseDir = ?config(case_top_dir, Config), application:stop(inets), application:load(inets), application:set_env(inets, services, [{httpc, {default, CaseDir}}]), application:start(inets), - http:set_options([{cookies, enabled}]), - {"cookie","$Version=0; test_cookie=true; $Path=/"} = http:cookie_header(?URL), + httpc:set_options([{cookies, enabled}]), + {"cookie","$Version=0; test_cookie=true; $Path=/"} = httpc:cookie_header(?URL), tsp("persistent_cookie -> Cookies 2: ~p", [httpc:which_cookies()]), ok. @@ -251,10 +259,10 @@ domain_cookie(Config) when is_list(Config) -> SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/;" "domain=.cookie.test.org"}], - http:verify_cookies(SetCookieHeaders, ?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), {"cookie","$Version=0; test_cookie=true; $Path=/; " "$Domain=.cookie.test.org"} = - http:cookie_header(?URL_DOMAIN), + httpc:cookie_header(?URL_DOMAIN), tsp("domain_cookie -> Cookies 2: ~p", [httpc:which_cookies()]), ok. @@ -275,8 +283,8 @@ secure_cookie(Config) when is_list(Config) -> tsp("secure_cookie -> Cookies 1: ~p", [httpc:which_cookies()]), SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/; secure"}], - tsp("secure_cookie -> verify cookies (1)"), - ok = http:verify_cookies(SetCookieHeaders, ?URL), + tsp("secure_cookie -> store cookies (1)"), + ok = httpc:store_cookies(SetCookieHeaders, ?URL), tsp("secure_cookie -> Cookies 2: ~p", [httpc:which_cookies()]), @@ -286,9 +294,9 @@ secure_cookie(Config) when is_list(Config) -> tsp("secure_cookie -> check cookie (plain)"), check_cookie("", ?URL), - tsp("secure_cookie -> verify cookies (2)"), + tsp("secure_cookie -> store cookies (2)"), SetCookieHeaders1 = [{"set-cookie", "test1_cookie=true; path=/; secure"}], - ok = http:verify_cookies(SetCookieHeaders1, ?URL), + ok = httpc:store_cookies(SetCookieHeaders1, ?URL), tsp("secure_cookie -> Cookies 3: ~p", [httpc:which_cookies()]), @@ -297,7 +305,7 @@ secure_cookie(Config) when is_list(Config) -> "test1_cookie=true; $Path=/", ?URL_SECURE), %% {"cookie","$Version=0; test_cookie=true; $Path=/; " -%% "test1_cookie=true; $Path=/"} = http:cookie_header(?URL_SECURE), +%% "test1_cookie=true; $Path=/"} = httpc:cookie_header(?URL_SECURE), tsp("secure_cookie -> Cookies 4: ~p", [httpc:which_cookies()]), @@ -305,38 +313,93 @@ secure_cookie(Config) when is_list(Config) -> tsp("secure_cookie -> done"), ok. +expect_cookie_header(No, ExpectedCookie) -> + case httpc:cookie_header(?URL) of + {"cookie", ExpectedCookie} -> + ok; + {"cookie", BadCookie} -> + io:format("Bad Cookie ~w: " + "~n Expected: ~s" + "~n Received: ~s" + "~n", [No, ExpectedCookie, BadCookie]), + exit({bad_cookie_header, No, ExpectedCookie, BadCookie}) + end. + +print_cookies(Pre) -> + io:format("~s: ", [Pre]), + print_cookies2(httpc:which_cookies()). + +print_cookies2([]) -> + ok; +print_cookies2([{cookies, Cookies}|Rest]) -> + print_cookies3("Cookies", Cookies), + print_cookies2(Rest); +print_cookies2([{session_cookies, Cookies}|Rest]) -> + print_cookies3("Session Cookies", Cookies), + print_cookies2(Rest); +print_cookies2([_|Rest]) -> + print_cookies2(Rest). + +print_cookies3(Header, []) -> + io:format(" ~s: []", [Header]); +print_cookies3(Header, Cookies) -> + io:format(" ~s: ", [Header]), + Prefix = " ", + PrintCookie = + fun(Cookie) -> + io:format("~s", [httpc_cookie:image_of(Prefix, Cookie)]) + end, + lists:foreach(PrintCookie, Cookies). + update_cookie(doc)-> - ["Test that a cookie can be updated."]; + ["Test that a (plain) cookie can be updated."]; update_cookie(suite) -> []; -update_cookie(Config) when is_list(Config)-> - SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/;" - "max-age=6500"}, - {"set-cookie", "test_cookie2=true; path=/;" - "max-age=6500"}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie", "$Version=0; test_cookie2=true; $Path=/; " - "test_cookie=true; $Path=/"} = http:cookie_header(?URL), - NewSetCookieHeaders = [{"set-cookie", "test_cookie=false; " - "path=/;max-age=6500"}], - http:verify_cookies(NewSetCookieHeaders, ?URL), - {"cookie", "$Version=0; test_cookie2=true; $Path=/; " - "test_cookie=false; $Path=/"} = http:cookie_header(?URL). - +update_cookie(Config) when is_list(Config) -> + print_cookies("Cookies before store"), + + SetCookieHeaders = + [{"set-cookie", "test_cookie=true; path=/; max-age=6500"}, + {"set-cookie", "test_cookie2=true; path=/; max-age=6500"}], + httpc:store_cookies(SetCookieHeaders, ?URL), + print_cookies("Cookies after first store"), + ExpectCookie1 = + "$Version=0; " + "test_cookie=true; $Path=/; " + "test_cookie2=true; $Path=/", + expect_cookie_header(1, ExpectCookie1), + + NewSetCookieHeaders = + [{"set-cookie", "test_cookie=false; path=/; max-age=6500"}], + httpc:store_cookies(NewSetCookieHeaders, ?URL), + print_cookies("Cookies after second store"), + ExpectCookie2 = + "$Version=0; " + "test_cookie2=true; $Path=/; " + "test_cookie=false; $Path=/", + expect_cookie_header(2, ExpectCookie2). + update_cookie_session(doc)-> - ["Test that a cookie can be updated."]; + ["Test that a session cookie can be updated."]; update_cookie_session(suite) -> []; update_cookie_session(Config) when is_list(Config)-> + print_cookies("Cookies before store"), + SetCookieHeaders = [{"set-cookie", "test_cookie=true; path=/"}, {"set-cookie", "test_cookie2=true; path=/"}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie", "$Version=0; test_cookie2=true; $Path=/; " - "test_cookie=true; $Path=/"} = http:cookie_header(?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), + print_cookies("Cookies after first store"), + ExpectedCookie1 = + "$Version=0; test_cookie=true; $Path=/; test_cookie2=true; $Path=/", + expect_cookie_header(1, ExpectedCookie1), + NewSetCookieHeaders = [{"set-cookie", "test_cookie=false; path=/"}], - http:verify_cookies(NewSetCookieHeaders, ?URL), - {"cookie", "$Version=0; test_cookie2=true; $Path=/; " - "test_cookie=false; $Path=/"} = http:cookie_header(?URL). + httpc:store_cookies(NewSetCookieHeaders, ?URL), + print_cookies("Cookies after second store"), + ExpectedCookie2 = + "$Version=0; test_cookie2=true; $Path=/; test_cookie=false; $Path=/", + expect_cookie_header(2, ExpectedCookie2). cookie_attributes(doc) -> @@ -348,8 +411,8 @@ cookie_attributes(Config) when is_list(Config) -> "comment=foobar; "%% Comment "foo=bar;" %% Nonsense should be ignored "max-age=60000"}], - http:verify_cookies(SetCookieHeaders, ?URL), - {"cookie","$Version=1; test_cookie=true"} = http:cookie_header(?URL), + httpc:store_cookies(SetCookieHeaders, ?URL), + {"cookie","$Version=1; test_cookie=true"} = httpc:cookie_header(?URL), ok. @@ -358,7 +421,7 @@ cookie_attributes(Config) when is_list(Config) -> %%-------------------------------------------------------------------- check_cookie(Expect, URL) -> - case http:cookie_header(URL) of + case httpc:cookie_header(URL) of {"cookie", Expect} -> ok; {"cookie", Unexpected} -> diff --git a/lib/inets/test/httpd_1_1.erl b/lib/inets/test/httpd_1_1.erl index 2a6110e3ea..07d94ea97a 100644 --- a/lib/inets/test/httpd_1_1.erl +++ b/lib/inets/test/httpd_1_1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,6 @@ %% -module(httpd_1_1). --author('[email protected]'). -include("test_server.hrl"). -include("test_server_line.hrl"). @@ -159,70 +158,79 @@ if_test(Type, Port, Host, Node, DocRoot)-> calendar:datetime_to_gregorian_seconds(FileInfo#file_info.mtime), Mod = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec-1)), - + CreatedSec-1)), + %% Test that we get the data when the file is modified ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ Host ++ - "\r\nIf-Modified-Since:" ++ - Mod ++ "\r\n\r\n", - [{statuscode, 200}]), - Mod1 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec+100)), - ok = httpd_test_lib:verify_request(Type,Host,Port,Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++"\r\nIf-Modified-Since:" - ++ Mod1 ++"\r\n\r\n", - [{statuscode, 304}]), + "GET / HTTP/1.1\r\nHost:" ++ Host ++ + "\r\nIf-Modified-Since:" ++ + Mod ++ "\r\n\r\n", + [{statuscode, 200}]), + Mod1 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( + CreatedSec+100)), + ok = httpd_test_lib:verify_request(Type,Host,Port,Node, + "GET / HTTP/1.1\r\nHost:" + ++ Host ++"\r\nIf-Modified-Since:" + ++ Mod1 ++"\r\n\r\n", + [{statuscode, 304}]), + + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, + "GET / HTTP/1.1\r\nHost:" ++ Host ++ + "\r\nIf-Modified-Since:" ++ + "AAA[...]AAAA" ++ "\r\n\r\n", + [{statuscode, 400}]), + + Mod2 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec+1)), + CreatedSec+1)), %% Control that the If-Unmodified-Header lmits the response ok = httpd_test_lib:verify_request(Type,Host,Port,Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:" ++ Mod2 - ++ "\r\n\r\n", - [{statuscode, 200}]), + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:" ++ Mod2 + ++ "\r\n\r\n", + [{statuscode, 200}]), Mod3 = httpd_util:rfc1123_date(calendar:gregorian_seconds_to_datetime( - CreatedSec-1)), + CreatedSec-1)), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ - "\r\nIf-Unmodified-Since:"++ Mod3 - ++"\r\n\r\n", - [{statuscode, 412}]), - + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ + "\r\nIf-Unmodified-Since:"++ Mod3 + ++"\r\n\r\n", + [{statuscode, 412}]), + %% Control that we get the body when the etag match ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ Host - ++"\r\n"++ - "If-Match:"++ - httpd_util:create_etag(FileInfo)++ - "\r\n\r\n", - [{statuscode, 200}]), + "GET / HTTP/1.1\r\nHost:" ++ Host + ++"\r\n"++ + "If-Match:"++ + httpd_util:create_etag(FileInfo)++ + "\r\n\r\n", + [{statuscode, 200}]), ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" ++ - Host ++ "\r\n"++ - "If-Match:NotEtag\r\n\r\n", - [{statuscode, 412}]), + "GET / HTTP/1.1\r\nHost:" ++ + Host ++ "\r\n"++ + "If-Match:NotEtag\r\n\r\n", + [{statuscode, 412}]), %% Control the response when the if-none-match header is there ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++"\r\n"++ - "If-None-Match:NoTaag," ++ - httpd_util:create_etag(FileInfo) ++ - "\r\n\r\n", - [{statuscode, 304}]), - + "GET / HTTP/1.1\r\nHost:" + ++ Host ++"\r\n"++ + "If-None-Match:NoTaag," ++ + httpd_util:create_etag(FileInfo) ++ + "\r\n\r\n", + [{statuscode, 304}]), + ok = httpd_test_lib:verify_request(Type, Host, Port, Node, - "GET / HTTP/1.1\r\nHost:" - ++ Host ++ "\r\n"++ - "If-None-Match:NotEtag," - "NeihterEtag\r\n\r\n", - [{statuscode,200}]). + "GET / HTTP/1.1\r\nHost:" + ++ Host ++ "\r\n"++ + "If-None-Match:NotEtag," + "NeihterEtag\r\n\r\n", + [{statuscode,200}]), + ok. http_trace(Type, Port, Host, Node)-> ok = httpd_test_lib:verify_request(Type, Host, Port, Node, diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index c4d4bf969b..ba31788ccc 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -68,127 +68,96 @@ -export([ pssl_mod_alias/1, - ossl_mod_alias/1, essl_mod_alias/1, pssl_mod_actions/1, - ossl_mod_actions/1, essl_mod_actions/1, pssl_mod_security/1, - ossl_mod_security/1, essl_mod_security/1, pssl_mod_auth/1, - ossl_mod_auth/1, essl_mod_auth/1, pssl_mod_auth_api/1, - ossl_mod_auth_api/1, essl_mod_auth_api/1, pssl_mod_auth_mnesia_api/1, - ossl_mod_auth_mnesia_api/1, essl_mod_auth_mnesia_api/1, pssl_mod_htaccess/1, - ossl_mod_htaccess/1, essl_mod_htaccess/1, pssl_mod_cgi/1, - ossl_mod_cgi/1, essl_mod_cgi/1, pssl_mod_esi/1, - ossl_mod_esi/1, essl_mod_esi/1, pssl_mod_get/1, - ossl_mod_get/1, essl_mod_get/1, pssl_mod_head/1, - ossl_mod_head/1, essl_mod_head/1, pssl_mod_all/1, - ossl_mod_all/1, essl_mod_all/1, pssl_load_light/1, - ossl_load_light/1, essl_load_light/1, pssl_load_medium/1, - ossl_load_medium/1, essl_load_medium/1, pssl_load_heavy/1, - ossl_load_heavy/1, essl_load_heavy/1, pssl_dos_hostname/1, - ossl_dos_hostname/1, essl_dos_hostname/1, pssl_time_test/1, - ossl_time_test/1, essl_time_test/1, pssl_restart_no_block/1, - ossl_restart_no_block/1, essl_restart_no_block/1, pssl_restart_disturbing_block/1, - ossl_restart_disturbing_block/1, essl_restart_disturbing_block/1, pssl_restart_non_disturbing_block/1, - ossl_restart_non_disturbing_block/1, essl_restart_non_disturbing_block/1, pssl_block_disturbing_idle/1, - ossl_block_disturbing_idle/1, essl_block_disturbing_idle/1, pssl_block_non_disturbing_idle/1, - ossl_block_non_disturbing_idle/1, essl_block_non_disturbing_idle/1, pssl_block_503/1, - ossl_block_503/1, essl_block_503/1, pssl_block_disturbing_active/1, - ossl_block_disturbing_active/1, essl_block_disturbing_active/1, pssl_block_non_disturbing_active/1, - ossl_block_non_disturbing_active/1, essl_block_non_disturbing_active/1, pssl_block_disturbing_active_timeout_not_released/1, - ossl_block_disturbing_active_timeout_not_released/1, essl_block_disturbing_active_timeout_not_released/1, pssl_block_disturbing_active_timeout_released/1, - ossl_block_disturbing_active_timeout_released/1, essl_block_disturbing_active_timeout_released/1, pssl_block_non_disturbing_active_timeout_not_released/1, - ossl_block_non_disturbing_active_timeout_not_released/1, essl_block_non_disturbing_active_timeout_not_released/1, pssl_block_non_disturbing_active_timeout_released/1, - ossl_block_non_disturbing_active_timeout_released/1, essl_block_non_disturbing_active_timeout_released/1, pssl_block_disturbing_blocker_dies/1, - ossl_block_disturbing_blocker_dies/1, essl_block_disturbing_blocker_dies/1, pssl_block_non_disturbing_blocker_dies/1, - ossl_block_non_disturbing_blocker_dies/1, essl_block_non_disturbing_blocker_dies/1 ]). @@ -272,8 +241,7 @@ groups() -> ip_block_non_disturbing_active_timeout_released, ip_block_disturbing_blocker_dies, ip_block_non_disturbing_blocker_dies]}, - {ssl, [], - [{group, pssl}, {group, ossl}, {group, essl}]}, + {ssl, [], [{group, pssl}, {group, essl}]}, {pssl, [], [pssl_mod_alias, pssl_mod_actions, pssl_mod_security, pssl_mod_auth, pssl_mod_auth_api, @@ -293,25 +261,6 @@ groups() -> pssl_block_non_disturbing_active_timeout_released, pssl_block_disturbing_blocker_dies, pssl_block_non_disturbing_blocker_dies]}, - {ossl, [], - [ossl_mod_alias, ossl_mod_actions, ossl_mod_security, - ossl_mod_auth, ossl_mod_auth_api, - ossl_mod_auth_mnesia_api, ossl_mod_htaccess, - ossl_mod_cgi, ossl_mod_esi, ossl_mod_get, ossl_mod_head, - ossl_mod_all, ossl_load_light, ossl_load_medium, - ossl_load_heavy, ossl_dos_hostname, ossl_time_test, - ossl_restart_no_block, ossl_restart_disturbing_block, - ossl_restart_non_disturbing_block, - ossl_block_disturbing_idle, - ossl_block_non_disturbing_idle, ossl_block_503, - ossl_block_disturbing_active, - ossl_block_non_disturbing_active, - ossl_block_disturbing_active_timeout_not_released, - ossl_block_disturbing_active_timeout_released, - ossl_block_non_disturbing_active_timeout_not_released, - ossl_block_non_disturbing_active_timeout_released, - ossl_block_disturbing_blocker_dies, - ossl_block_non_disturbing_blocker_dies]}, {essl, [], [essl_mod_alias, essl_mod_actions, essl_mod_security, essl_mod_auth, essl_mod_auth_api, @@ -493,7 +442,6 @@ init_per_testcase2(Case, Config) -> [X, $s, $s, $l | _] -> case X of $p -> ssl; - $o -> ossl; $e -> essl end; _ -> @@ -636,7 +584,6 @@ init_per_testcase3(Case, Config) -> SslTag = case X of $p -> ssl; % plain - $o -> ossl; % OpenSSL based ssl $e -> essl % Erlang based ssl end, case inets_test_lib:start_http_server_ssl( @@ -646,14 +593,13 @@ init_per_testcase3(Case, Config) -> ok -> "mod_htaccess"; Other -> - error_logger:info_report("Other: ~p~n", [Other]), + error_logger:info_msg("Other: ~p~n", [Other]), {skip, "SSL does not seem to be supported"} end; [X, $s, $s, $l, $_ | Rest] -> SslTag = case X of $p -> ssl; - $o -> ossl; $e -> essl end, case inets_test_lib:start_http_server_ssl( @@ -663,7 +609,7 @@ init_per_testcase3(Case, Config) -> ok -> Rest; Other -> - error_logger:info_report("Other: ~p~n", [Other]), + error_logger:info_msg("Other: ~p~n", [Other]), {skip, "SSL does not seem to be supported"} end; "ipv6_" ++ _ = TestCaseStr -> @@ -740,6 +686,19 @@ end_per_testcase2(Case, Config) -> %%------------------------------------------------------------------------- +http_1_1_ip(doc) -> + ["HTTP/1.1"]; +http_1_1_ip(suite) -> + [ + ip_host, + ip_chunked, + ip_expect, + ip_range, + ip_if_test, + ip_http_trace, + ip_http1_1_head, + ip_mod_cgi_chunked_encoding_test + ]. %%------------------------------------------------------------------------- @@ -1158,13 +1117,6 @@ pssl_mod_alias(suite) -> pssl_mod_alias(Config) when is_list(Config) -> ssl_mod_alias(ssl, Config). -ossl_mod_alias(doc) -> - ["Module test: mod_alias - using new of configure old SSL"]; -ossl_mod_alias(suite) -> - []; -ossl_mod_alias(Config) when is_list(Config) -> - ssl_mod_alias(ossl, Config). - essl_mod_alias(doc) -> ["Module test: mod_alias - using new of configure new SSL"]; essl_mod_alias(suite) -> @@ -1188,13 +1140,6 @@ pssl_mod_actions(suite) -> pssl_mod_actions(Config) when is_list(Config) -> ssl_mod_actions(ssl, Config). -ossl_mod_actions(doc) -> - ["Module test: mod_actions - using new of configure old SSL"]; -ossl_mod_actions(suite) -> - []; -ossl_mod_actions(Config) when is_list(Config) -> - ssl_mod_actions(ossl, Config). - essl_mod_actions(doc) -> ["Module test: mod_actions - using new of configure new SSL"]; essl_mod_actions(suite) -> @@ -1220,13 +1165,6 @@ pssl_mod_security(suite) -> pssl_mod_security(Config) when is_list(Config) -> ssl_mod_security(ssl, Config). -ossl_mod_security(doc) -> - ["Module test: mod_security - using new of configure old SSL"]; -ossl_mod_security(suite) -> - []; -ossl_mod_security(Config) when is_list(Config) -> - ssl_mod_security(ossl, Config). - essl_mod_security(doc) -> ["Module test: mod_security - using new of configure new SSL"]; essl_mod_security(suite) -> @@ -1253,13 +1191,6 @@ pssl_mod_auth(suite) -> pssl_mod_auth(Config) when is_list(Config) -> ssl_mod_auth(ssl, Config). -ossl_mod_auth(doc) -> - ["Module test: mod_auth - using new of configure old SSL"]; -ossl_mod_auth(suite) -> - []; -ossl_mod_auth(Config) when is_list(Config) -> - ssl_mod_auth(ossl, Config). - essl_mod_auth(doc) -> ["Module test: mod_auth - using new of configure new SSL"]; essl_mod_auth(suite) -> @@ -1284,13 +1215,6 @@ pssl_mod_auth_api(suite) -> pssl_mod_auth_api(Config) when is_list(Config) -> ssl_mod_auth_api(ssl, Config). -ossl_mod_auth_api(doc) -> - ["Module test: mod_auth - using new of configure old SSL"]; -ossl_mod_auth_api(suite) -> - []; -ossl_mod_auth_api(Config) when is_list(Config) -> - ssl_mod_auth_api(ossl, Config). - essl_mod_auth_api(doc) -> ["Module test: mod_auth - using new of configure new SSL"]; essl_mod_auth_api(suite) -> @@ -1317,13 +1241,6 @@ pssl_mod_auth_mnesia_api(suite) -> pssl_mod_auth_mnesia_api(Config) when is_list(Config) -> ssl_mod_auth_mnesia_api(ssl, Config). -ossl_mod_auth_mnesia_api(doc) -> - ["Module test: mod_auth_mnesia_api - using new of configure old SSL"]; -ossl_mod_auth_mnesia_api(suite) -> - []; -ossl_mod_auth_mnesia_api(Config) when is_list(Config) -> - ssl_mod_auth_mnesia_api(ossl, Config). - essl_mod_auth_mnesia_api(doc) -> ["Module test: mod_auth_mnesia_api - using new of configure new SSL"]; essl_mod_auth_mnesia_api(suite) -> @@ -1348,13 +1265,6 @@ pssl_mod_htaccess(suite) -> pssl_mod_htaccess(Config) when is_list(Config) -> ssl_mod_htaccess(ssl, Config). -ossl_mod_htaccess(doc) -> - ["Module test: mod_htaccess - using new of configure old SSL"]; -ossl_mod_htaccess(suite) -> - []; -ossl_mod_htaccess(Config) when is_list(Config) -> - ssl_mod_htaccess(ossl, Config). - essl_mod_htaccess(doc) -> ["Module test: mod_htaccess - using new of configure new SSL"]; essl_mod_htaccess(suite) -> @@ -1379,13 +1289,6 @@ pssl_mod_cgi(suite) -> pssl_mod_cgi(Config) when is_list(Config) -> ssl_mod_cgi(ssl, Config). -ossl_mod_cgi(doc) -> - ["Module test: mod_cgi - using new of configure old SSL"]; -ossl_mod_cgi(suite) -> - []; -ossl_mod_cgi(Config) when is_list(Config) -> - ssl_mod_cgi(ossl, Config). - essl_mod_cgi(doc) -> ["Module test: mod_cgi - using new of configure new SSL"]; essl_mod_cgi(suite) -> @@ -1415,13 +1318,6 @@ pssl_mod_esi(suite) -> pssl_mod_esi(Config) when is_list(Config) -> ssl_mod_esi(ssl, Config). -ossl_mod_esi(doc) -> - ["Module test: mod_esi - using new of configure old SSL"]; -ossl_mod_esi(suite) -> - []; -ossl_mod_esi(Config) when is_list(Config) -> - ssl_mod_esi(ossl, Config). - essl_mod_esi(doc) -> ["Module test: mod_esi - using new of configure new SSL"]; essl_mod_esi(suite) -> @@ -1446,13 +1342,6 @@ pssl_mod_get(suite) -> pssl_mod_get(Config) when is_list(Config) -> ssl_mod_get(ssl, Config). -ossl_mod_get(doc) -> - ["Module test: mod_get - using new of configure old SSL"]; -ossl_mod_get(suite) -> - []; -ossl_mod_get(Config) when is_list(Config) -> - ssl_mod_get(ossl, Config). - essl_mod_get(doc) -> ["Module test: mod_get - using new of configure new SSL"]; essl_mod_get(suite) -> @@ -1477,13 +1366,6 @@ pssl_mod_head(suite) -> pssl_mod_head(Config) when is_list(Config) -> ssl_mod_head(ssl, Config). -ossl_mod_head(doc) -> - ["Module test: mod_head - using new of configure old SSL"]; -ossl_mod_head(suite) -> - []; -ossl_mod_head(Config) when is_list(Config) -> - ssl_mod_head(ossl, Config). - essl_mod_head(doc) -> ["Module test: mod_head - using new of configure new SSL"]; essl_mod_head(suite) -> @@ -1508,13 +1390,6 @@ pssl_mod_all(suite) -> pssl_mod_all(Config) when is_list(Config) -> ssl_mod_all(ssl, Config). -ossl_mod_all(doc) -> - ["All modules test - using new of configure old SSL"]; -ossl_mod_all(suite) -> - []; -ossl_mod_all(Config) when is_list(Config) -> - ssl_mod_all(ossl, Config). - essl_mod_all(doc) -> ["All modules test - using new of configure new SSL"]; essl_mod_all(suite) -> @@ -1539,13 +1414,6 @@ pssl_load_light(suite) -> pssl_load_light(Config) when is_list(Config) -> ssl_load_light(ssl, Config). -ossl_load_light(doc) -> - ["Test light load - using new of configure old SSL"]; -ossl_load_light(suite) -> - []; -ossl_load_light(Config) when is_list(Config) -> - ssl_load_light(ossl, Config). - essl_load_light(doc) -> ["Test light load - using new of configure new SSL"]; essl_load_light(suite) -> @@ -1571,13 +1439,6 @@ pssl_load_medium(suite) -> pssl_load_medium(Config) when is_list(Config) -> ssl_load_medium(ssl, Config). -ossl_load_medium(doc) -> - ["Test medium load - using new of configure old SSL"]; -ossl_load_medium(suite) -> - []; -ossl_load_medium(Config) when is_list(Config) -> - ssl_load_medium(ossl, Config). - essl_load_medium(doc) -> ["Test medium load - using new of configure new SSL"]; essl_load_medium(suite) -> @@ -1609,13 +1470,6 @@ pssl_load_heavy(suite) -> pssl_load_heavy(Config) when is_list(Config) -> ssl_load_heavy(ssl, Config). -ossl_load_heavy(doc) -> - ["Test heavy load - using new of configure old SSL"]; -ossl_load_heavy(suite) -> - []; -ossl_load_heavy(Config) when is_list(Config) -> - ssl_load_heavy(ossl, Config). - essl_load_heavy(doc) -> ["Test heavy load - using new of configure new SSL"]; essl_load_heavy(suite) -> @@ -1647,13 +1501,6 @@ pssl_dos_hostname(suite) -> pssl_dos_hostname(Config) when is_list(Config) -> ssl_dos_hostname(ssl, Config). -ossl_dos_hostname(doc) -> - ["Denial Of Service (DOS) attack test case - using new of configure old SSL"]; -ossl_dos_hostname(suite) -> - []; -ossl_dos_hostname(Config) when is_list(Config) -> - ssl_dos_hostname(ossl, Config). - essl_dos_hostname(doc) -> ["Denial Of Service (DOS) attack test case - using new of configure new SSL"]; essl_dos_hostname(suite) -> @@ -1679,13 +1526,6 @@ pssl_time_test(suite) -> pssl_time_test(Config) when is_list(Config) -> ssl_time_test(ssl, Config). -ossl_time_test(doc) -> - ["using new of configure old SSL"]; -ossl_time_test(suite) -> - []; -ossl_time_test(Config) when is_list(Config) -> - ssl_time_test(ossl, Config). - essl_time_test(doc) -> ["using new of configure new SSL"]; essl_time_test(suite) -> @@ -1725,14 +1565,6 @@ pssl_block_503(suite) -> pssl_block_503(Config) when is_list(Config) -> ssl_block_503(ssl, Config). -ossl_block_503(doc) -> - ["Check that you will receive status code 503 when the server" - " is blocked and 200 when its not blocked - using new of configure old SSL."]; -ossl_block_503(suite) -> - []; -ossl_block_503(Config) when is_list(Config) -> - ssl_block_503(ossl, Config). - essl_block_503(doc) -> ["Check that you will receive status code 503 when the server" " is blocked and 200 when its not blocked - using new of configure new SSL."]; @@ -1760,15 +1592,6 @@ pssl_block_disturbing_idle(suite) -> pssl_block_disturbing_idle(Config) when is_list(Config) -> ssl_block_disturbing_idle(ssl, Config). -ossl_block_disturbing_idle(doc) -> - ["Check that you can block/unblock an idle server. The strategy " - "distribing does not really make a difference in this case." - "Using new of configure old SSL"]; -ossl_block_disturbing_idle(suite) -> - []; -ossl_block_disturbing_idle(Config) when is_list(Config) -> - ssl_block_disturbing_idle(ossl, Config). - essl_block_disturbing_idle(doc) -> ["Check that you can block/unblock an idle server. The strategy " "distribing does not really make a difference in this case." @@ -1797,15 +1620,6 @@ pssl_block_non_disturbing_idle(suite) -> pssl_block_non_disturbing_idle(Config) when is_list(Config) -> ssl_block_non_disturbing_idle(ssl, Config). -ossl_block_non_disturbing_idle(doc) -> - ["Check that you can block/unblock an idle server. The strategy " - "non distribing does not really make a difference in this case." - "Using new of configure old SSL"]; -ossl_block_non_disturbing_idle(suite) -> - []; -ossl_block_non_disturbing_idle(Config) when is_list(Config) -> - ssl_block_non_disturbing_idle(ossl, Config). - essl_block_non_disturbing_idle(doc) -> ["Check that you can block/unblock an idle server. The strategy " "non distribing does not really make a difference in this case." @@ -1834,15 +1648,6 @@ pssl_block_disturbing_active(suite) -> pssl_block_disturbing_active(Config) when is_list(Config) -> ssl_block_disturbing_active(ssl, Config). -ossl_block_disturbing_active(doc) -> - ["Check that you can block/unblock an active server. The strategy " - "distribing means ongoing requests should be terminated." - "Using new of configure old SSL"]; -ossl_block_disturbing_active(suite) -> - []; -ossl_block_disturbing_active(Config) when is_list(Config) -> - ssl_block_disturbing_active(ossl, Config). - essl_block_disturbing_active(doc) -> ["Check that you can block/unblock an active server. The strategy " "distribing means ongoing requests should be terminated." @@ -1871,15 +1676,6 @@ pssl_block_non_disturbing_active(suite) -> pssl_block_non_disturbing_active(Config) when is_list(Config) -> ssl_block_non_disturbing_active(ssl, Config). -ossl_block_non_disturbing_active(doc) -> - ["Check that you can block/unblock an idle server. The strategy " - "non distribing means the ongoing requests should be compleated." - "Using new of configure old SSL"]; -ossl_block_non_disturbing_active(suite) -> - []; -ossl_block_non_disturbing_active(Config) when is_list(Config) -> - ssl_block_non_disturbing_active(ossl, Config). - essl_block_non_disturbing_active(doc) -> ["Check that you can block/unblock an idle server. The strategy " "non distribing means the ongoing requests should be compleated." @@ -1910,17 +1706,6 @@ pssl_block_disturbing_active_timeout_not_released(Config) when is_list(Config) -> ssl_block_disturbing_active_timeout_not_released(ssl, Config). -ossl_block_disturbing_active_timeout_not_released(doc) -> - ["Check that you can block an active server. The strategy " - "distribing means ongoing requests should be compleated" - "if the timeout does not occur." - "Using new of configure old SSL"]; -ossl_block_disturbing_active_timeout_not_released(suite) -> - []; -ossl_block_disturbing_active_timeout_not_released(Config) - when is_list(Config) -> - ssl_block_disturbing_active_timeout_not_released(ossl, Config). - essl_block_disturbing_active_timeout_not_released(doc) -> ["Check that you can block an active server. The strategy " "distribing means ongoing requests should be compleated" @@ -1954,17 +1739,6 @@ pssl_block_disturbing_active_timeout_released(Config) when is_list(Config) -> ssl_block_disturbing_active_timeout_released(ssl, Config). -ossl_block_disturbing_active_timeout_released(doc) -> - ["Check that you can block an active server. The strategy " - "distribing means ongoing requests should be terminated when" - "the timeout occurs." - "Using new of configure old SSL"]; -ossl_block_disturbing_active_timeout_released(suite) -> - []; -ossl_block_disturbing_active_timeout_released(Config) - when is_list(Config) -> - ssl_block_disturbing_active_timeout_released(ossl, Config). - essl_block_disturbing_active_timeout_released(doc) -> ["Check that you can block an active server. The strategy " "distribing means ongoing requests should be terminated when" @@ -1999,16 +1773,6 @@ pssl_block_non_disturbing_active_timeout_not_released(Config) when is_list(Config) -> ssl_block_non_disturbing_active_timeout_not_released(ssl, Config). -ossl_block_non_disturbing_active_timeout_not_released(doc) -> - ["Check that you can block an active server. The strategy " - "non non distribing means ongoing requests should be completed." - "Using new of configure old SSL"]; -ossl_block_non_disturbing_active_timeout_not_released(suite) -> - []; -ossl_block_non_disturbing_active_timeout_not_released(Config) - when is_list(Config) -> - ssl_block_non_disturbing_active_timeout_not_released(ossl, Config). - essl_block_non_disturbing_active_timeout_not_released(doc) -> ["Check that you can block an active server. The strategy " "non non distribing means ongoing requests should be completed." @@ -2043,17 +1807,6 @@ pssl_block_non_disturbing_active_timeout_released(Config) when is_list(Config) -> ssl_block_non_disturbing_active_timeout_released(ssl, Config). -ossl_block_non_disturbing_active_timeout_released(doc) -> - ["Check that you can block an active server. The strategy " - "non distribing means ongoing requests should be completed. " - "When the timeout occurs the block operation sohould be canceled." - "Using new of configure old SSL"]; -ossl_block_non_disturbing_active_timeout_released(suite) -> - []; -ossl_block_non_disturbing_active_timeout_released(Config) - when is_list(Config) -> - ssl_block_non_disturbing_active_timeout_released(ossl, Config). - essl_block_non_disturbing_active_timeout_released(doc) -> ["Check that you can block an active server. The strategy " "non distribing means ongoing requests should be completed. " @@ -2087,13 +1840,6 @@ pssl_block_disturbing_blocker_dies(suite) -> pssl_block_disturbing_blocker_dies(Config) when is_list(Config) -> ssl_block_disturbing_blocker_dies(ssl, Config). -ossl_block_disturbing_blocker_dies(doc) -> - ["using new of configure old SSL"]; -ossl_block_disturbing_blocker_dies(suite) -> - []; -ossl_block_disturbing_blocker_dies(Config) when is_list(Config) -> - ssl_block_disturbing_blocker_dies(ossl, Config). - essl_block_disturbing_blocker_dies(doc) -> ["using new of configure new SSL"]; essl_block_disturbing_blocker_dies(suite) -> @@ -2118,13 +1864,6 @@ pssl_block_non_disturbing_blocker_dies(suite) -> pssl_block_non_disturbing_blocker_dies(Config) when is_list(Config) -> ssl_block_non_disturbing_blocker_dies(ssl, Config). -ossl_block_non_disturbing_blocker_dies(doc) -> - ["using new of configure old SSL"]; -ossl_block_non_disturbing_blocker_dies(suite) -> - []; -ossl_block_non_disturbing_blocker_dies(Config) when is_list(Config) -> - ssl_block_non_disturbing_blocker_dies(ossl, Config). - essl_block_non_disturbing_blocker_dies(doc) -> ["using new of configure new SSL"]; essl_block_non_disturbing_blocker_dies(suite) -> @@ -2149,13 +1888,6 @@ pssl_restart_no_block(suite) -> pssl_restart_no_block(Config) when is_list(Config) -> ssl_restart_no_block(ssl, Config). -ossl_restart_no_block(doc) -> - ["using new of configure old SSL"]; -ossl_restart_no_block(suite) -> - []; -ossl_restart_no_block(Config) when is_list(Config) -> - ssl_restart_no_block(ossl, Config). - essl_restart_no_block(doc) -> ["using new of configure new SSL"]; essl_restart_no_block(suite) -> @@ -2180,13 +1912,6 @@ pssl_restart_disturbing_block(suite) -> pssl_restart_disturbing_block(Config) when is_list(Config) -> ssl_restart_disturbing_block(ssl, Config). -ossl_restart_disturbing_block(doc) -> - ["using new of configure old SSL"]; -ossl_restart_disturbing_block(suite) -> - []; -ossl_restart_disturbing_block(Config) when is_list(Config) -> - ssl_restart_disturbing_block(ossl, Config). - essl_restart_disturbing_block(doc) -> ["using new of configure new SSL"]; essl_restart_disturbing_block(suite) -> @@ -2244,13 +1969,6 @@ pssl_restart_non_disturbing_block(suite) -> pssl_restart_non_disturbing_block(Config) when is_list(Config) -> ssl_restart_non_disturbing_block(ssl, Config). -ossl_restart_non_disturbing_block(doc) -> - ["using new of configure old SSL"]; -ossl_restart_non_disturbing_block(suite) -> - []; -ossl_restart_non_disturbing_block(Config) when is_list(Config) -> - ssl_restart_non_disturbing_block(ossl, Config). - essl_restart_non_disturbing_block(doc) -> ["using new of configure new SSL"]; essl_restart_non_disturbing_block(suite) -> @@ -2571,24 +2289,24 @@ ticket_5913(doc) -> ["Tests that a header without last-modified is handled"]; ticket_5913(suite) -> []; ticket_5913(Config) -> - ok=httpd_test_lib:verify_request(ip_comm, ?config(host, Config), - ?IP_PORT, ?config(node, Config), + ok = httpd_test_lib:verify_request(ip_comm, ?config(host, Config), + ?IP_PORT, ?config(node, Config), "GET /cgi-bin/erl/httpd_example:get_bin " "HTTP/1.0\r\n\r\n", [{statuscode, 200}, - {version, "HTTP/1.0"}]), + {version, "HTTP/1.0"}]), ok. ticket_6003(doc) -> ["Tests that a URI with a bad hexadecimal code is handled"]; ticket_6003(suite) -> []; ticket_6003(Config) -> - ok=httpd_test_lib:verify_request(ip_comm, ?config(host, Config), - ?IP_PORT, ?config(node, Config), - "GET http://www.erlang.org/%skalle " - "HTTP/1.0\r\n\r\n", - [{statuscode, 400}, - {version, "HTTP/1.0"}]), + ok = httpd_test_lib:verify_request(ip_comm, ?config(host, Config), + ?IP_PORT, ?config(node, Config), + "GET http://www.erlang.org/%skalle " + "HTTP/1.0\r\n\r\n", + [{statuscode, 400}, + {version, "HTTP/1.0"}]), ok. ticket_7304(doc) -> @@ -2646,7 +2364,6 @@ create_config(Config, Access, FileName) -> SSL = if (Type =:= ssl) orelse - (Type =:= ossl) orelse (Type =:= essl) -> [cline(["SSLCertificateFile ", filename:join(ServerRoot, "ssl/ssl_server.pem")]), @@ -3041,7 +2758,6 @@ create_ipv6_config(Config, FileName, Ipv6Address) -> SSL = if (SockType =:= ssl) orelse - (SockType =:= ossl) orelse (SockType =:= essl) -> [cline(["SSLCertificateFile ", filename:join(ServerRoot, "ssl/ssl_server.pem")]), diff --git a/lib/inets/test/httpd_basic_SUITE.erl b/lib/inets/test/httpd_basic_SUITE.erl index f23d0b4765..4cd38f2ec4 100644 --- a/lib/inets/test/httpd_basic_SUITE.erl +++ b/lib/inets/test/httpd_basic_SUITE.erl @@ -59,9 +59,28 @@ init_per_suite(Config) -> "~n Config: ~p", [Config]), ok = inets:start(), PrivDir = ?config(priv_dir, Config), - HttpdConf = [{port, 0}, {ipfamily, inet}, - {server_name, "httpd_test"}, {server_root, PrivDir}, - {document_root, PrivDir}, {bind_address, "localhost"}], + + Dummy = +"<HTML> +<HEAD> +<TITLE>/index.html</TITLE> +</HEAD> +<BODY> +DUMMY +</BODY> +</HTML>", + + DummyFile = filename:join([PrivDir,"dummy.html"]), + {ok, Fd} = file:open(DummyFile, [write]), + ok = file:write(Fd, Dummy), + ok = file:close(Fd), + HttpdConf = [{port, 0}, + {ipfamily, inet}, + {server_name, "httpd_test"}, + {server_root, PrivDir}, + {document_root, PrivDir}, + {bind_address, "localhost"}], + [{httpd_conf, HttpdConf} | Config]. %%-------------------------------------------------------------------- @@ -133,6 +152,10 @@ uri_too_long_414(Config) when is_list(Config) -> {version, "HTTP/0.9"}]), inets:stop(httpd, Pid). + +%%------------------------------------------------------------------------- +%%------------------------------------------------------------------------- + header_too_long_413(doc) -> ["Test that too long headers's get 413 HTTP code"]; header_too_long_413(suite) -> @@ -152,34 +175,92 @@ header_too_long_413(Config) when is_list(Config) -> {version, "HTTP/1.1"}]), inets:stop(httpd, Pid). + +%%------------------------------------------------------------------------- +%%------------------------------------------------------------------------- + escaped_url_in_error_body(doc) -> ["Test Url-encoding see OTP-8940"]; escaped_url_in_error_body(suite) -> []; escaped_url_in_error_body(Config) when is_list(Config) -> - tsp("escaped_url_in_error_body -> entry with" - "~n Config: ~p", [Config]), - HttpdConf = ?config(httpd_conf, Config), - {ok, Pid} = inets:start(httpd, [{port, 0} | HttpdConf]), - Info = httpd:info(Pid), - Port = proplists:get_value(port, Info), - _Address = proplists:get_value(bind_address, Info), - Path = "/<b>this_is_bold</b>", - URL = ?URL_START ++ integer_to_list(Port) ++ Path, - EscapedPath = http_uri:encode(Path), - {ok, {404, Body1}} = httpc:request(get, {URL, []}, - [{url_encode, true}, - {version, "HTTP/1.0"}], - [{full_result, false}]), - EscapedPath = find_URL_path(string:tokens(Body1, " ")), - {ok, {404, Body2}} = httpc:request(get, {URL, []}, - [{url_encode, false}, - {version, "HTTP/1.0"}], - [{full_result, false}]), + tsp("escaped_url_in_error_body -> entry"), + HttpdConf = ?config(httpd_conf, Config), + {ok, Pid} = inets:start(httpd, [{port, 0} | HttpdConf]), + Info = httpd:info(Pid), + Port = proplists:get_value(port, Info), + _Address = proplists:get_value(bind_address, Info), + + %% Request 1 + tsp("escaped_url_in_error_body -> request 1"), + URL1 = ?URL_START ++ integer_to_list(Port), + %% Make sure the server is ok, by making a request for a valid page + case httpc:request(get, {URL1 ++ "/dummy.html", []}, + [{url_encode, false}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {200, _}} -> + %% Don't care about the the body, just that we get a ok response + ok; + {ok, UnexpectedOK1} -> + tsf({unexpected_ok_1, UnexpectedOK1}) + end, + + %% Request 2 + tsp("escaped_url_in_error_body -> request 2"), + %% Make sure the server is ok, by making a request for a valid page + case httpc:request(get, {URL1 ++ "/dummy.html", []}, + [{url_encode, true}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {200, _}} -> + %% Don't care about the the body, just that we get a ok response + ok; + {ok, UnexpectedOK2} -> + tsf({unexpected_ok_2, UnexpectedOK2}) + end, + + %% Request 3 + tsp("escaped_url_in_error_body -> request 3"), + %% Ask for a non-existing page(1) + Path = "/<b>this_is_bold<b>", HTMLEncodedPath = http_util:html_encode(Path), - HTMLEncodedPath = find_URL_path(string:tokens(Body2, " ")), + URL2 = URL1 ++ Path, + case httpc:request(get, {URL2, []}, + [{url_encode, true}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {404, Body3}} -> + case find_URL_path(string:tokens(Body3, " ")) of + HTMLEncodedPath -> + ok; + BadPath3 -> + tsf({unexpected_path_3, HTMLEncodedPath, BadPath3}) + end; + {ok, UnexpectedOK3} -> + tsf({unexpected_ok_1, UnexpectedOK3}) + end, + + %% Request 4 + tsp("escaped_url_in_error_body -> request 4"), + %% Ask for a non-existing page(2) + case httpc:request(get, {URL2, []}, + [{url_encode, false}, + {version, "HTTP/1.0"}], + [{full_result, false}]) of + {ok, {404, Body4}} -> + case find_URL_path(string:tokens(Body4, " ")) of + HTMLEncodedPath -> + ok; + BadPath4 -> + tsf({unexpected_path_2, HTMLEncodedPath, BadPath4}) + end; + {ok, UnexpectedOK4} -> + tsf({unexpected_ok_4, UnexpectedOK4}) + end, + tsp("escaped_url_in_error_body -> stop inets"), inets:stop(httpd, Pid), - tsp("escaped_url_in_error_body -> done"), + tsp("escaped_url_in_error_body -> done"), ok. find_URL_path([]) -> @@ -191,7 +272,14 @@ find_URL_path([_ | Rest]) -> tsp(F) -> - tsp(F, []). + inets_test_lib:tsp(F). tsp(F, A) -> - test_server:format("~p ~p:" ++ F ++ "~n", [self(), ?MODULE | A]). + inets_test_lib:tsp(F, A). + +tsf(Reason) -> + test_server:fail(Reason). + + +skip(Reason) -> + {skip, Reason}. diff --git a/lib/inets/test/httpd_mod.erl b/lib/inets/test/httpd_mod.erl index 1754cec7bc..5016cdb9e6 100644 --- a/lib/inets/test/httpd_mod.erl +++ b/lib/inets/test/httpd_mod.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 2005-2011. All Rights Reserved. -%% +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the @@ -88,13 +88,13 @@ actions(Type, Port, Host, Node) -> %%------------------------------------------------------------------------- security(ServerRoot, Type, Port, Host, Node) -> -%% io:format(user, "~w:security -> entry with" -%% "~n ServerRoot: ~p" -%% "~n Type: ~p" -%% "~n Port: ~p" -%% "~n Host: ~p" -%% "~n Node: ~p" -%% "~n", [?MODULE, ServerRoot, Type, Port, Host, Node]), + %% io:format(user, "~w:security -> entry with" + %% "~n ServerRoot: ~p" + %% "~n Type: ~p" + %% "~n Port: ~p" + %% "~n Host: ~p" + %% "~n Node: ~p" + %% "~n", [?MODULE, ServerRoot, Type, Port, Host, Node]), %% io:format(user, "~w:security -> register~n", [?MODULE]), global:register_name(mod_security_test, self()), % Receive events @@ -175,8 +175,8 @@ security(ServerRoot, Type, Port, Host, Node) -> [{"one",_, Port, OpenDir,_}] -> ok; Blocked -> - io:format(user, "~w:security -> Blocked: ~p" - "~n", [?MODULE, Blocked]), + %% io:format(user, "~w:security -> Blocked: ~p" + %% "~n", [?MODULE, Blocked]), exit({unexpected_blocked, Blocked}) end, @@ -917,11 +917,11 @@ list_users(Node, Root, _Host, Port, Dir) -> receive_security_event(Event, Node, Port) -> -%% io:format(user, "~w:receive_security_event -> entry with" -%% "~n Event: ~p" -%% "~n Node: ~p" -%% "~n Port: ~p" -%% "~n", [?MODULE, Event, Node, Port]), + %% io:format(user, "~w:receive_security_event -> entry with" + %% "~n Event: ~p" + %% "~n Node: ~p" + %% "~n Port: ~p" + %% "~n", [?MODULE, Event, Node, Port]), receive Event -> ok; diff --git a/lib/inets/test/httpd_test_lib.erl b/lib/inets/test/httpd_test_lib.erl index 2903aaafa5..1c7bb512cc 100644 --- a/lib/inets/test/httpd_test_lib.erl +++ b/lib/inets/test/httpd_test_lib.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2001-2010. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2001-2011. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the @@ -140,6 +140,9 @@ request(#state{mfa = {Module, Function, Args}, HeadRequest = lists:sublist(RequestStr, 1, 4), receive {tcp, Socket, Data} -> + io:format("~p ~w[~w]request -> received (tcp) data" + "~n Data: ~p" + "~n", [self(), ?MODULE, ?LINE, Data]), print(tcp, Data, State), case Module:Function([Data | Args]) of {ok, Parsed} -> @@ -150,11 +153,19 @@ request(#state{mfa = {Module, Function, Args}, request(State#state{mfa = NewMFA}, TimeOut) end; {tcp_closed, Socket} when Function =:= whole_body -> + io:format("~p ~w[~w]request -> " + "received (tcp) closed when whole_body" + "~n", [self(), ?MODULE, ?LINE]), print(tcp, "closed", State), State#state{body = hd(Args)}; {tcp_closed, Socket} -> + io:format("~p ~w[~w]request -> received (tcp) closed" + "~n", [self(), ?MODULE, ?LINE]), test_server:fail(connection_closed); {tcp_error, Socket, Reason} -> + io:format("~p ~w[~w]request -> received (tcp) error" + "~n Reason: ~p" + "~n", [self(), ?MODULE, ?LINE, Reason]), test_server:fail({tcp_error, Reason}); {ssl, Socket, Data} -> print(ssl, Data, State), @@ -174,11 +185,21 @@ request(#state{mfa = {Module, Function, Args}, {ssl_error, Socket, Reason} -> test_server:fail({ssl_error, Reason}) after TimeOut -> + io:format("~p ~w[~w]request -> timeout" + "~n", [self(), ?MODULE, ?LINE]), test_server:fail(connection_timed_out) end. handle_http_msg({Version, StatusCode, ReasonPharse, Headers, Body}, State = #state{request = RequestStr}) -> + io:format("~p ~w[~w]handle_http_msg -> entry with" + "~n Version: ~p" + "~n StatusCode: ~p" + "~n ReasonPharse: ~p" + "~n Headers: ~p" + "~n Body: ~p" + "~n", [self(), ?MODULE, ?LINE, + Version, StatusCode, ReasonPharse, Headers, Body]), case is_expect(RequestStr) of true -> State#state{status_line = {Version, @@ -235,13 +256,14 @@ handle_http_body(Body, State = #state{headers = Headers, end. validate(RequestStr, #state{status_line = {Version, StatusCode, _}, - headers = Headers, - body = Body}, Options, N, P) -> + headers = Headers, + body = Body}, Options, N, P) -> %% tsp("validate -> entry with" %% "~n StatusCode: ~p" %% "~n Headers: ~p" %% "~n Body: ~p", [StatusCode, Headers, Body]), + check_version(Version, Options), case lists:keysearch(statuscode, 1, Options) of {value, _} -> @@ -255,6 +277,7 @@ validate(RequestStr, #state{status_line = {Version, StatusCode, _}, list_to_integer(Headers#http_response_h.'content-length'), Body). + %%-------------------------------------------------------------------- %% Internal functions %%------------------------------------------------------------------ @@ -263,21 +286,20 @@ check_version(Version, Options) -> {value, {version, Version}} -> ok; {value, {version, Ver}} -> - test_server:fail({wrong_version, [{got, Version}, - {expected, Ver}]}); + tsf({wrong_version, [{got, Version}, + {expected, Ver}]}); _ -> case Version of "HTTP/1.1" -> ok; _ -> - test_server:fail({wrong_version, [{got, Version}, - {expected, "HTTP/1.1"}]}) + tsf({wrong_version, [{got, Version}, + {expected, "HTTP/1.1"}]}) end end. check_status_code(StatusCode, [], Options) -> - test_server:fail({wrong_status_code, [{got, StatusCode}, - {expected, Options}]}); + tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}); check_status_code(StatusCode, Current = [_ | Rest], Options) -> case lists:keysearch(statuscode, 1, Current) of {value, {statuscode, StatusCode}} -> @@ -285,8 +307,7 @@ check_status_code(StatusCode, Current = [_ | Rest], Options) -> {value, {statuscode, _OtherStatus}} -> check_status_code(StatusCode, Rest, Options); false -> - test_server:fail({wrong_status_code, [{got, StatusCode}, - {expected, Options}]}) + tsf({wrong_status_code, [{got, StatusCode}, {expected, Options}]}) end. do_validate(_, [], _, _) -> @@ -317,8 +338,7 @@ do_validate(Header, [{header, HeaderField, Value}|Rest],N,P) -> Header}) end, do_validate(Header, Rest, N, P); -do_validate(Header,[{no_last_modified,HeaderField}|Rest],N,P) -> -% io:format("Header: ~p~nHeaderField: ~p~n",[Header,HeaderField]), +do_validate(Header,[{no_last_modified, HeaderField}|Rest],N,P) -> case lists:keysearch(HeaderField,1,Header) of {value,_} -> test_server:fail({wrong_header_field_value, HeaderField, @@ -331,7 +351,6 @@ do_validate(Header, [_Unknown | Rest], N, P) -> do_validate(Header, Rest, N, P). is_expect(RequestStr) -> - case inets_regexp:match(RequestStr, "xpect:100-continue") of {match, _, _}-> true; @@ -340,15 +359,15 @@ is_expect(RequestStr) -> end. %% OTP-5775, content-length -check_body("GET /cgi-bin/erl/httpd_example:get_bin HTTP/1.0\r\n\r\n", 200, "text/html", Length, _Body) when Length /= 274-> - test_server:fail(content_length_error); +check_body("GET /cgi-bin/erl/httpd_example:get_bin HTTP/1.0\r\n\r\n", 200, "text/html", Length, _Body) when (Length =/= 274) -> + tsf(content_length_error); check_body("GET /cgi-bin/cgi_echo HTTP/1.0\r\n\r\n", 200, "text/plain", _, Body) -> case size(Body) of 100 -> ok; _ -> - test_server:fail(content_length_error) + tsf(content_length_error) end; check_body(RequestStr, 200, "text/html", _, Body) -> diff --git a/lib/inets/test/httpd_time_test.erl b/lib/inets/test/httpd_time_test.erl index f39f9faff0..c54674be36 100644 --- a/lib/inets/test/httpd_time_test.erl +++ b/lib/inets/test/httpd_time_test.erl @@ -19,7 +19,7 @@ %% -module(httpd_time_test). --export([t/3, t1/2, t2/2, t3/2, t4/2]). +-export([t/3, t1/2, t2/2, t4/2]). -export([do/1, do/2, do/3, do/4, do/5]). @@ -45,10 +45,6 @@ t2(Host, Port) -> t(ssl, Host, Port). -t3(Host, Port) -> - t(ossl, Host, Port). - - t4(Host, Port) -> t(essl, Host, Port). diff --git a/lib/inets/test/inets_test_lib.erl b/lib/inets/test/inets_test_lib.erl index 2e19c41f16..ddb1a49394 100644 --- a/lib/inets/test/inets_test_lib.erl +++ b/lib/inets/test/inets_test_lib.erl @@ -340,9 +340,6 @@ connect_bin(SockType, Host, Port) -> connect_bin(ssl, Host, Port, Opts0) -> Opts = [binary, {packet,0} | Opts0], connect(ssl, Host, Port, Opts); -connect_bin(ossl, Host, Port, Opts0) -> - Opts = [{ssl_imp, old}, binary, {packet,0} | Opts0], - connect(ssl, Host, Port, Opts); connect_bin(essl, Host, Port, Opts0) -> Opts = [{ssl_imp, new}, binary, {packet,0}, {reuseaddr, true} | Opts0], connect(ssl, Host, Port, Opts); @@ -357,9 +354,6 @@ connect_byte(SockType, Host, Port) -> connect_byte(ssl, Host, Port, Opts0) -> Opts = [{packet,0} | Opts0], connect(ssl, Host, Port, Opts); -connect_byte(ossl, Host, Port, Opts0) -> - Opts = [{ssl_imp, old}, {packet,0} | Opts0], - connect(ssl, Host, Port, Opts); connect_byte(essl, Host, Port, Opts0) -> Opts = [{ssl_imp, new}, {packet,0} | Opts0], connect(ssl, Host, Port, Opts); @@ -421,8 +415,6 @@ connect(ip_comm, Host, Port, Opts) -> send(ssl, Socket, Data) -> ssl:send(Socket, Data); -send(ossl, Socket, Data) -> - ssl:send(Socket, Data); send(essl, Socket, Data) -> ssl:send(Socket, Data); send(ip_comm,Socket,Data) -> @@ -431,8 +423,6 @@ send(ip_comm,Socket,Data) -> close(ssl,Socket) -> catch ssl:close(Socket); -close(ossl,Socket) -> - catch ssl:close(Socket); close(essl,Socket) -> catch ssl:close(Socket); close(ip_comm,Socket) -> diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 4abc1733d3..1df4558e45 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 5.7 +INETS_VSN = 5.8 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" |