Update structured headers implementation to RFC 8941

I have in the process changed the output a little with
regard to parameters. The dictionaries also build as
a list of key-values now to keep ordering. This should
be the definitive interface.

3 files changed, 240 insertions, 137 deletions

diff --git a/Makefile b/Makefile
index f67c726..ecc635b 100644
--- a/Makefile
+++ b/Makefile
@@ -21,11 +21,12 @@ LOCAL_DEPS = crypto
 DOC_DEPS = asciideck
 
 TEST_DEPS = $(if $(CI_ERLANG_MK),ci.erlang.mk) base32 horse proper jsx \
-	structured-header-tests uritemplate-tests
+	decimal structured-header-tests uritemplate-tests
 dep_base32 = git https://github.com/dnsimple/base32_erlang main
 dep_horse = git https://github.com/ninenines/horse.git master
 dep_jsx = git https://github.com/talentdeficit/jsx v2.10.0
-dep_structured-header-tests = git https://github.com/httpwg/structured-header-tests e614583397e7f65e0082c0fff3929f32a298b9f2
+dep_decimal = git https://github.com/egobrain/decimal 0.6.2
+dep_structured-header-tests = git https://github.com/httpwg/structured-header-tests main
 dep_uritemplate-tests = git https://github.com/uri-templates/uritemplate-test master
 
 # CI configuration.
diff --git a/src/cow_http_hd.erl b/src/cow_http_hd.erl
index e2a0a1d..a24c171 100644
--- a/src/cow_http_hd.erl
+++ b/src/cow_http_hd.erl
@@ -1,4 +1,4 @@
-%% Copyright (c) 2014-2018, Loïc Hoguin <[email protected]>
+%% Copyright (c) 2014-2022, Loïc Hoguin <[email protected]>
 %%
 %% Permission to use, copy, modify, and/or distribute this software for any
 %% purpose with or without fee is hereby granted, provided that the above
@@ -3227,11 +3227,11 @@ parse_upgrade_error_test_() ->
 parse_variant_key(VariantKey, NumMembers) ->
 	List = cow_http_struct_hd:parse_list(VariantKey),
 	[case Inner of
-		{with_params, InnerList, #{}} ->
+		{list, InnerList, []} ->
 			NumMembers = length(InnerList),
 			[case Item of
-				{with_params, {token, Value}, #{}} -> Value;
-				{with_params, {string, Value}, #{}} -> Value
+				{item, {token, Value}, []} -> Value;
+				{item, {string, Value}, []} -> Value
 			end || Item <- InnerList]
 	end || Inner <- List].
 
@@ -3261,9 +3261,9 @@ parse_variant_key_error_test_() ->
 %% We assume that the lists are of correct length.
 variant_key(VariantKeys) ->
 	cow_http_struct_hd:list([
-		{with_params, [
-			{with_params, {string, Value}, #{}}
-		|| Value <- InnerList], #{}}
+		{list, [
+			{item, {string, Value}, []}
+		|| Value <- InnerList], []}
 	|| InnerList <- VariantKeys]).
 
 -ifdef(TEST).
@@ -3287,14 +3287,14 @@ variant_key_identity_test_() ->
 
 -spec parse_variants(binary()) -> [{binary(), [binary()]}].
 parse_variants(Variants) ->
-	{Dict0, Order} = cow_http_struct_hd:parse_dictionary(Variants),
-	Dict = maps:map(fun(_, {with_params, List, #{}}) ->
-		[case Item of
-			{with_params, {token, Value}, #{}} -> Value;
-			{with_params, {string, Value}, #{}} -> Value
-		end || Item <- List]
-	end, Dict0),
-	[{Key, maps:get(Key, Dict)} || Key <- Order].
+	Dict = cow_http_struct_hd:parse_dictionary(Variants),
+	[case DictItem of
+		{Key, {list, List, []}} ->
+			{Key, [case Item of
+				{item, {token, Value}, []} -> Value;
+				{item, {string, Value}, []} -> Value
+			end || Item <- List]}
+	end || DictItem <- Dict].
 
 -ifdef(TEST).
 parse_variants_test_() ->
@@ -3317,9 +3317,9 @@ parse_variants_test_() ->
 -spec variants([{binary(), [binary()]}]) -> iolist().
 variants(Variants) ->
 	cow_http_struct_hd:dictionary([
-		{Key, {with_params, [
-			{with_params, {string, Value}, #{}}
-		|| Value <- List], #{}}}
+		{Key, {list, [
+			{item, {string, Value}, []}
+		|| Value <- List], []}}
 	|| {Key, List} <- Variants]).
 
 -ifdef(TEST).
diff --git a/src/cow_http_struct_hd.erl b/src/cow_http_struct_hd.erl
index 373c8da..c19f9a0 100644
--- a/src/cow_http_struct_hd.erl
+++ b/src/cow_http_struct_hd.erl
@@ -1,4 +1,4 @@
-%% Copyright (c) 2019, Loïc Hoguin <[email protected]>
+%% Copyright (c) 2019-2022, Loïc Hoguin <[email protected]>
 %%
 %% Permission to use, copy, modify, and/or distribute this software for any
 %% purpose with or without fee is hereby granted, provided that the above
@@ -15,17 +15,18 @@
 %% The mapping between Erlang and structured headers types is as follow:
 %%
 %% List: list()
-%% Dictionary: map()
+%% Inner list: {list, [item()], params()}
+%% Dictionary: [{binary(), item()}]
+%%   There is no distinction between empty list and empty dictionary.
+%% Item with parameters: {item, bare_item(), params()}
+%% Parameters: [{binary(), bare_item()}]
 %% Bare item: one bare_item() that can be of type:
 %% Integer: integer()
-%% Float: float()
+%% Decimal: {decimal, {integer(), integer()}}
 %% String: {string, binary()}
 %% Token: {token, binary()}
 %% Byte sequence: {binary, binary()}
 %% Boolean: boolean()
-%% And finally:
-%% Type with Parameters: {with_params, Type, Parameters}
-%% Parameters: [{binary(), bare_item()}]
 
 -module(cow_http_struct_hd).
 
@@ -39,13 +40,13 @@
 -include("cow_parse.hrl").
 
 -type sh_list() :: [sh_item() | sh_inner_list()].
--type sh_inner_list() :: sh_with_params([sh_item()]).
--type sh_params() :: #{binary() => sh_bare_item() | undefined}.
--type sh_dictionary() :: {#{binary() => sh_item() | sh_inner_list()}, [binary()]}.
--type sh_item() :: sh_with_params(sh_bare_item()).
--type sh_bare_item() :: integer() | float() | boolean()
+-type sh_inner_list() :: {list, [sh_item()], sh_params()}.
+-type sh_params() :: [{binary(), sh_bare_item()}].
+-type sh_dictionary() :: [{binary(), sh_item() | sh_inner_list()}].
+-type sh_item() :: {item, sh_bare_item(), sh_params()}.
+-type sh_bare_item() :: integer() | sh_decimal() | boolean()
 	| {string | token | binary, binary()}.
--type sh_with_params(Type) :: {with_params, Type, sh_params()}.
+-type sh_decimal() :: {decimal, {integer(), integer()}}.
 
 -define(IS_LC_ALPHA(C),
 	(C =:= $a) or (C =:= $b) or (C =:= $c) or (C =:= $d) or (C =:= $e) or
@@ -60,35 +61,41 @@
 
 -spec parse_dictionary(binary()) -> sh_dictionary().
 parse_dictionary(<<>>) ->
-	{#{}, []};
-parse_dictionary(<<C,R/bits>>) when ?IS_LC_ALPHA(C) ->
-	{Dict, Order, <<>>} = parse_dict_key(R, #{}, [], <<C>>),
-	{Dict, Order}.
+	[];
+parse_dictionary(<<C,R/bits>>) when ?IS_LC_ALPHA(C) or (C =:= $*) ->
+	parse_dict_key(R, [], <<C>>).
 
-parse_dict_key(<<$=,$(,R0/bits>>, Acc, Order, K) ->
-	false = maps:is_key(K, Acc),
+parse_dict_key(<<$=,$(,R0/bits>>, Acc, K) ->
 	{Item, R} = parse_inner_list(R0, []),
-	parse_dict_before_sep(R, Acc#{K => Item}, [K|Order]);
-parse_dict_key(<<$=,R0/bits>>, Acc, Order, K) ->
-	false = maps:is_key(K, Acc),
+	parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, Item}));
+parse_dict_key(<<$=,R0/bits>>, Acc, K) ->
 	{Item, R} = parse_item1(R0),
-	parse_dict_before_sep(R, Acc#{K => Item}, [K|Order]);
-parse_dict_key(<<C,R/bits>>, Acc, Order, K)
+	parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, Item}));
+parse_dict_key(<<C,R/bits>>, Acc, K)
 		when ?IS_LC_ALPHA(C) or ?IS_DIGIT(C)
-			or (C =:= $_) or (C =:= $-) or (C =:= $*) ->
-	parse_dict_key(R, Acc, Order, <<K/binary,C>>).
-
-parse_dict_before_sep(<<C,R/bits>>, Acc, Order) when ?IS_WS(C) ->
-	parse_dict_before_sep(R, Acc, Order);
-parse_dict_before_sep(<<C,R/bits>>, Acc, Order) when C =:= $, ->
-	parse_dict_before_member(R, Acc, Order);
-parse_dict_before_sep(<<>>, Acc, Order) ->
-	{Acc, lists:reverse(Order), <<>>}.
-
-parse_dict_before_member(<<C,R/bits>>, Acc, Order) when ?IS_WS(C) ->
-	parse_dict_before_member(R, Acc, Order);
-parse_dict_before_member(<<C,R/bits>>, Acc, Order) when ?IS_LC_ALPHA(C) ->
-	parse_dict_key(R, Acc, Order, <<C>>).
+			or (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) ->
+	parse_dict_key(R, Acc, <<K/binary,C>>);
+parse_dict_key(<<$;,R0/bits>>, Acc, K) ->
+	{Params, R} = parse_before_param(R0, []),
+	parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, {item, true, Params}}));
+parse_dict_key(R, Acc, K) ->
+	parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, {item, true, []}})).
+
+parse_dict_before_sep(<<$\s,R/bits>>, Acc) ->
+	parse_dict_before_sep(R, Acc);
+parse_dict_before_sep(<<$\t,R/bits>>, Acc) ->
+	parse_dict_before_sep(R, Acc);
+parse_dict_before_sep(<<C,R/bits>>, Acc) when C =:= $, ->
+	parse_dict_before_member(R, Acc);
+parse_dict_before_sep(<<>>, Acc) ->
+	Acc.
+
+parse_dict_before_member(<<$\s,R/bits>>, Acc) ->
+	parse_dict_before_member(R, Acc);
+parse_dict_before_member(<<$\t,R/bits>>, Acc) ->
+	parse_dict_before_member(R, Acc);
+parse_dict_before_member(<<C,R/bits>>, Acc) when ?IS_LC_ALPHA(C) or (C =:= $*) ->
+	parse_dict_key(R, Acc, <<C>>).
 
 -spec parse_item(binary()) -> sh_item().
 parse_item(Bin) ->
@@ -98,10 +105,10 @@ parse_item(Bin) ->
 parse_item1(Bin) ->
 	case parse_bare_item(Bin) of
 		{Item, <<$;,R/bits>>} ->
-			{Params, Rest} = parse_before_param(R, #{}),
-			{{with_params, Item, Params}, Rest};
+			{Params, Rest} = parse_before_param(R, []),
+			{{item, Item, Params}, Rest};
 		{Item, Rest} ->
-			{{with_params, Item, #{}}, Rest}
+			{{item, Item, []}, Rest}
 	end.
 
 -spec parse_list(binary()) -> sh_list().
@@ -117,86 +124,104 @@ parse_list_member(R0, Acc) ->
 	{Item, R} = parse_item1(R0),
 	parse_list_before_sep(R, [Item|Acc]).
 
-parse_list_before_sep(<<C,R/bits>>, Acc) when ?IS_WS(C) ->
+parse_list_before_sep(<<$\s,R/bits>>, Acc) ->
+	parse_list_before_sep(R, Acc);
+parse_list_before_sep(<<$\t,R/bits>>, Acc) ->
 	parse_list_before_sep(R, Acc);
 parse_list_before_sep(<<$,,R/bits>>, Acc) ->
 	parse_list_before_member(R, Acc);
 parse_list_before_sep(<<>>, Acc) ->
 	lists:reverse(Acc).
 
-parse_list_before_member(<<C,R/bits>>, Acc) when ?IS_WS(C) ->
+parse_list_before_member(<<$\s,R/bits>>, Acc) ->
+	parse_list_before_member(R, Acc);
+parse_list_before_member(<<$\t,R/bits>>, Acc) ->
 	parse_list_before_member(R, Acc);
 parse_list_before_member(R, Acc) ->
 	parse_list_member(R, Acc).
 
 %% Internal.
 
-parse_inner_list(<<C,R/bits>>, Acc) when ?IS_WS(C) ->
+parse_inner_list(<<$\s,R/bits>>, Acc) ->
 	parse_inner_list(R, Acc);
 parse_inner_list(<<$),$;,R0/bits>>, Acc) ->
-	{Params, R} = parse_before_param(R0, #{}),
-	{{with_params, lists:reverse(Acc), Params}, R};
+	{Params, R} = parse_before_param(R0, []),
+	{{list, lists:reverse(Acc), Params}, R};
 parse_inner_list(<<$),R/bits>>, Acc) ->
-	{{with_params, lists:reverse(Acc), #{}}, R};
+	{{list, lists:reverse(Acc), []}, R};
 parse_inner_list(R0, Acc) ->
 	{Item, R = <<C,_/bits>>} = parse_item1(R0),
 	true = (C =:= $\s) orelse (C =:= $)),
 	parse_inner_list(R, [Item|Acc]).
 
-parse_before_param(<<C,R/bits>>, Acc) when ?IS_WS(C) ->
+parse_before_param(<<$\s,R/bits>>, Acc) ->
 	parse_before_param(R, Acc);
-parse_before_param(<<C,R/bits>>, Acc) when ?IS_LC_ALPHA(C) ->
+parse_before_param(<<C,R/bits>>, Acc) when ?IS_LC_ALPHA(C) or (C =:= $*) ->
 	parse_param(R, Acc, <<C>>).
 
 parse_param(<<$;,R/bits>>, Acc, K) ->
-	parse_before_param(R, Acc#{K => undefined});
+	parse_before_param(R, lists:keystore(K, 1, Acc, {K, true}));
 parse_param(<<$=,R0/bits>>, Acc, K) ->
 	case parse_bare_item(R0) of
 		{Item, <<$;,R/bits>>} ->
-			false = maps:is_key(K, Acc),
-			parse_before_param(R, Acc#{K => Item});
+			parse_before_param(R, lists:keystore(K, 1, Acc, {K, Item}));
 		{Item, R} ->
-			false = maps:is_key(K, Acc),
-			{Acc#{K => Item}, R}
+			{lists:keystore(K, 1, Acc, {K, Item}), R}
 	end;
 parse_param(<<C,R/bits>>, Acc, K)
 		when ?IS_LC_ALPHA(C) or ?IS_DIGIT(C)
-			or (C =:= $_) or (C =:= $-) or (C =:= $*) ->
+			or (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) ->
 	parse_param(R, Acc, <<K/binary,C>>);
 parse_param(R, Acc, K) ->
-	false = maps:is_key(K, Acc),
-	{Acc#{K => undefined}, R}.
+	{lists:keystore(K, 1, Acc, {K, true}), R}.
 
-%% Integer or float.
+%% Integer or decimal.
 parse_bare_item(<<$-,R/bits>>) -> parse_number(R, 0, <<$->>);
 parse_bare_item(<<C,R/bits>>) when ?IS_DIGIT(C) -> parse_number(R, 1, <<C>>);
 %% String.
 parse_bare_item(<<$",R/bits>>) -> parse_string(R, <<>>);
 %% Token.
-parse_bare_item(<<C,R/bits>>) when ?IS_ALPHA(C) -> parse_token(R, <<C>>);
+parse_bare_item(<<C,R/bits>>) when ?IS_ALPHA(C) or (C =:= $*) -> parse_token(R, <<C>>);
 %% Byte sequence.
-parse_bare_item(<<$*,R/bits>>) -> parse_binary(R, <<>>);
+parse_bare_item(<<$:,R/bits>>) -> parse_binary(R, <<>>);
 %% Boolean.
 parse_bare_item(<<"?0",R/bits>>) -> {false, R};
 parse_bare_item(<<"?1",R/bits>>) -> {true, R}.
 
 parse_number(<<C,R/bits>>, L, Acc) when ?IS_DIGIT(C) ->
 	parse_number(R, L+1, <<Acc/binary,C>>);
-parse_number(<<C,R/bits>>, L, Acc) when C =:= $. ->
-	parse_float(R, L, 0, <<Acc/binary,C>>);
+parse_number(<<$.,R/bits>>, L, Acc) ->
+	parse_decimal(R, L, 0, Acc, <<>>);
 parse_number(R, L, Acc) when L =< 15 ->
 	{binary_to_integer(Acc), R}.
 
-parse_float(<<C,R/bits>>, L1, L2, Acc) when ?IS_DIGIT(C) ->
-	parse_float(R, L1, L2+1, <<Acc/binary,C>>);
-parse_float(R, L1, L2, Acc) when
-		L1 =< 9, L2 =< 6;
-		L1 =< 10, L2 =< 5;
-		L1 =< 11, L2 =< 4;
-		L1 =< 12, L2 =< 3;
-		L1 =< 13, L2 =< 2;
-		L1 =< 14, L2 =< 1 ->
-	{binary_to_float(Acc), R}.
+parse_decimal(<<C,R/bits>>, L1, L2, IntAcc, FracAcc) when ?IS_DIGIT(C) ->
+	parse_decimal(R, L1, L2+1, IntAcc, <<FracAcc/binary,C>>);
+parse_decimal(R, L1, L2, IntAcc, FracAcc0) when L1 =< 12, L2 >= 1, L2 =< 3 ->
+	%% While not strictly required this gives a more consistent representation.
+	FracAcc = case FracAcc0 of
+		<<$0>> -> <<>>;
+		<<$0,$0>> -> <<>>;
+		<<$0,$0,$0>> -> <<>>;
+		<<A,B,$0>> -> <<A,B>>;
+		<<A,$0,$0>> -> <<A>>;
+		<<A,$0>> -> <<A>>;
+		_ -> FracAcc0
+	end,
+	Mul = case byte_size(FracAcc) of
+		3 -> 1000;
+		2 -> 100;
+		1 -> 10;
+		0 -> 1
+	end,
+	Int = binary_to_integer(IntAcc),
+	Frac = case FracAcc of
+		<<>> -> 0;
+		%% Mind the sign.
+		_ when Int < 0 -> -binary_to_integer(FracAcc);
+		_ -> binary_to_integer(FracAcc)
+	end,
+	{{decimal, {Int * Mul + Frac, -byte_size(FracAcc)}}, R}.
 
 parse_string(<<$\\,$",R/bits>>, Acc) ->
 	parse_string(R, <<Acc/binary,$">>);
@@ -215,7 +240,7 @@ parse_token(<<C,R/bits>>, Acc) when ?IS_TOKEN(C) or (C =:= $:) or (C =:= $/) ->
 parse_token(R, Acc) ->
 	{{token, Acc}, R}.
 
-parse_binary(<<$*,R/bits>>, Acc) ->
+parse_binary(<<$:,R/bits>>, Acc) ->
 	{{binary, base64:decode(Acc)}, R};
 parse_binary(<<C,R/bits>>, Acc) when ?IS_ALPHANUM(C) or (C =:= $+) or (C =:= $/) or (C =:= $=) ->
 	parse_binary(R, <<Acc/binary,C>>).
@@ -231,10 +256,13 @@ parse_struct_hd_test_() ->
 				%% The implementation is strict. We fail whenever we can.
 				CanFail = maps:get(<<"can_fail">>, Test, false),
 				MustFail = maps:get(<<"must_fail">>, Test, false),
+				io:format("must fail ~p~nexpected json ~0p~n",
+					[MustFail, maps:get(<<"expected">>, Test, undefined)]),
 				Expected = case MustFail of
 					true -> undefined;
 					false -> expected_to_term(maps:get(<<"expected">>, Test))
 				end,
+				io:format("expected term: ~0p", [Expected]),
 				Raw = raw_to_binary(Raw0),
 				case HeaderType of
 					<<"dictionary">> when MustFail; CanFail ->
@@ -251,7 +279,7 @@ parse_struct_hd_test_() ->
 					<<"list">> when MustFail; CanFail ->
 						{'EXIT', _} = (catch parse_list(Raw));
 					<<"dictionary">> ->
-						{Expected, _Order} = (catch parse_dictionary(Raw));
+						Expected = (catch parse_dictionary(Raw));
 					<<"item">> ->
 						Expected = (catch parse_item(Raw));
 					<<"list">> ->
@@ -265,26 +293,45 @@ parse_struct_hd_test_() ->
 		} <- Tests]
 	end || File <- Files]).
 
+%% The tests JSON use arrays for almost everything. Identifying
+%% what is what requires looking deeper in the values:
+%%
+%% dict: [["k", v], ["k2", v2]] (values may have params)
+%% params: [["k", v], ["k2", v2]] (no params for values)
+%% list: [e1, e2, e3]
+%% inner-list: [[ [items...], params]]
+%% item: [bare, params]
+
 %% Item.
-expected_to_term(E=[_, Params]) when is_map(Params) ->
-	e2t(E);
+expected_to_term([Bare, []])
+		when is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) ->
+	{item, e2tb(Bare), []};
+expected_to_term([Bare, Params = [[<<_/bits>>, _]|_]])
+		when is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) ->
+	{item, e2tb(Bare), e2tp(Params)};
+%% Empty list or dictionary.
+expected_to_term([]) ->
+	[];
+%% Dictionary.
+%%
+%% We exclude empty list from values because that could
+%% be confused with an outer list of strings. There is
+%% currently no conflicts in the tests thankfully.
+expected_to_term(Dict = [[<<_/bits>>, V]|_]) when V =/= [] ->
+	e2t(Dict);
 %% Outer list.
-expected_to_term(Expected) when is_list(Expected) ->
-	[e2t(E) || E <- Expected];
-expected_to_term(Expected) ->
-	e2t(Expected).
+expected_to_term(List) when is_list(List) ->
+	[e2t(E) || E <- List].
 
 %% Dictionary.
-e2t(Dict) when is_map(Dict) ->
-	maps:map(fun(_, V) -> e2t(V) end, Dict);
+e2t(Dict = [[<<_/bits>>, _]|_]) ->
+	[{K, e2t(V)} || [K, V] <- Dict];
 %% Inner list.
 e2t([List, Params]) when is_list(List) ->
-	{with_params, [e2t(E) || E <- List],
-		maps:map(fun(_, P) -> e2tb(P) end, Params)};
+	{list, [e2t(E) || E <- List], e2tp(Params)};
 %% Item.
 e2t([Bare, Params]) ->
-	{with_params, e2tb(Bare),
-		maps:map(fun(_, P) -> e2tb(P) end, Params)}.
+	{item, e2tb(Bare), e2tp(Params)}.
 
 %% Bare item.
 e2tb(#{<<"__type">> := <<"token">>, <<"value">> := V}) ->
@@ -293,11 +340,18 @@ e2tb(#{<<"__type">> := <<"binary">>, <<"value">> := V}) ->
 	{binary, base32:decode(V)};
 e2tb(V) when is_binary(V) ->
 	{string, V};
-e2tb(null) ->
-	undefined;
+e2tb(V) when is_float(V) ->
+	%% There should be no rounding needed for the test cases.
+	{decimal, decimal:to_decimal(V, #{precision => 3, rounding => round_down})};
 e2tb(V) ->
 	V.
 
+%% Params.
+e2tp([]) ->
+	[];
+e2tp(Params) ->
+	[{K, e2tb(V)} || [K, V] <- Params].
+
 %% The Cowlib parsers currently do not support resuming parsing
 %% in the case of multiple headers. To make tests work we modify
 %% the raw value the same way Cowboy does when encountering
@@ -308,7 +362,7 @@ e2tb(V) ->
 raw_to_binary(RawList) ->
 	trim_ws(iolist_to_binary(lists:join(<<", ">>, RawList))).
 
-trim_ws(<<C,R/bits>>) when ?IS_WS(C) -> trim_ws(R);
+trim_ws(<<$\s,R/bits>>) -> trim_ws(R);
 trim_ws(R) -> trim_ws_end(R, byte_size(R) - 1).
 
 trim_ws_end(_, -1) ->
@@ -316,7 +370,6 @@ trim_ws_end(_, -1) ->
 trim_ws_end(Value, N) ->
 	case binary:at(Value, N) of
 		$\s -> trim_ws_end(Value, N - 1);
-		$\t -> trim_ws_end(Value, N - 1);
 		_ ->
 			S = N + 1,
 			<< Value2:S/binary, _/bits >> = Value,
@@ -326,71 +379,118 @@ trim_ws_end(Value, N) ->
 
 %% Building.
 
--spec dictionary(#{binary() => sh_item() | sh_inner_list()}
-		| [{binary(), sh_item() | sh_inner_list()}])
+-spec dictionary(#{binary() => sh_item() | sh_inner_list()} | sh_dictionary())
 	-> iolist().
-%% @todo Also accept this? dictionary({Map, Order}) ->
 dictionary(Map) when is_map(Map) ->
 	dictionary(maps:to_list(Map));
 dictionary(KVList) when is_list(KVList) ->
 	lists:join(<<", ">>, [
-		[Key, $=, item_or_inner_list(Value)]
+		case Value of
+			true -> Key;
+			_ -> [Key, $=, item_or_inner_list(Value)]
+		end
 	|| {Key, Value} <- KVList]).
 
 -spec item(sh_item()) -> iolist().
-item({with_params, BareItem, Params}) ->
+item({item, BareItem, Params}) ->
 	[bare_item(BareItem), params(Params)].
 
 -spec list(sh_list()) -> iolist().
 list(List) ->
 	lists:join(<<", ">>, [item_or_inner_list(Value) || Value <- List]).
 
-item_or_inner_list(Value={with_params, List, _}) when is_list(List) ->
+item_or_inner_list(Value = {list, _, _}) ->
 	inner_list(Value);
 item_or_inner_list(Value) ->
 	item(Value).
 
-inner_list({with_params, List, Params}) ->
+inner_list({list, List, Params}) ->
 	[$(, lists:join($\s, [item(Value) || Value <- List]), $), params(Params)].
 
 bare_item({string, String}) ->
 	[$", escape_string(String, <<>>), $"];
+%% @todo Must fail if Token has invalid characters.
 bare_item({token, Token}) ->
 	Token;
 bare_item({binary, Binary}) ->
-	[$*, base64:encode(Binary), $*];
+	[$:, base64:encode(Binary), $:];
+bare_item({decimal, {Base, Exp}}) when Exp >= 0 ->
+	Mul = case Exp of
+		0 -> 1;
+		1 -> 10;
+		2 -> 100;
+		3 -> 1000;
+		4 -> 10000;
+		5 -> 100000;
+		6 -> 1000000;
+		7 -> 10000000;
+		8 -> 100000000;
+		9 -> 1000000000;
+		10 -> 10000000000;
+		11 -> 100000000000;
+		12 -> 1000000000000
+	end,
+	MaxLenWithSign = if
+		Base < 0 -> 13;
+		true -> 12
+	end,
+	Bin = integer_to_binary(Base * Mul),
+	true = byte_size(Bin) =< MaxLenWithSign,
+	[Bin, <<".0">>];
+bare_item({decimal, {Base, -1}}) ->
+	Int = Base div 10,
+	Frac = abs(Base) rem 10,
+	[integer_to_binary(Int), $., integer_to_binary(Frac)];
+bare_item({decimal, {Base, -2}}) ->
+	Int = Base div 100,
+	Frac = abs(Base) rem 100,
+	[integer_to_binary(Int), $., integer_to_binary(Frac)];
+bare_item({decimal, {Base, -3}}) ->
+	Int = Base div 1000,
+	Frac = abs(Base) rem 1000,
+	[integer_to_binary(Int), $., integer_to_binary(Frac)];
+bare_item({decimal, {Base, Exp}}) ->
+	Div = exp_div(Exp),
+	Int0 = Base div Div,
+	true = abs(Int0) < 1000000000000,
+	Frac0 = abs(Base) rem Div,
+	DivFrac = Div div 1000,
+	Frac1 = Frac0 div DivFrac,
+	{Int, Frac} = if
+		(Frac0 rem DivFrac) > (DivFrac div 2) ->
+			case Frac1 of
+				999 when Int0 < 0 -> {Int0 - 1, 0};
+				999 -> {Int0 + 1, 0};
+				_ -> {Int0, Frac1 + 1}
+			end;
+		true ->
+			{Int0, Frac1}
+	end,
+	[integer_to_binary(Int), $., if
+		Frac < 10 -> [$0, $0, integer_to_binary(Frac)];
+		Frac < 100 -> [$0, integer_to_binary(Frac)];
+		true -> integer_to_binary(Frac)
+	end];
 bare_item(Integer) when is_integer(Integer) ->
 	integer_to_binary(Integer);
-%% In order to properly reproduce the float as a string we
-%% must first determine how many decimals we want in the
-%% fractional component, otherwise rounding errors may occur.
-bare_item(Float) when is_float(Float) ->
-	Decimals = case trunc(Float) of
-		I when I >= 10000000000000 -> 1;
-		I when I >= 1000000000000 -> 2;
-		I when I >= 100000000000 -> 3;
-		I when I >= 10000000000 -> 4;
-		I when I >= 1000000000 -> 5;
-		_ -> 6
-	end,
-	float_to_binary(Float, [{decimals, Decimals}, compact]);
 bare_item(true) ->
 	<<"?1">>;
 bare_item(false) ->
 	<<"?0">>.
 
+exp_div(0) -> 1;
+exp_div(N) -> 10 * exp_div(N + 1).
+
 escape_string(<<>>, Acc) -> Acc;
 escape_string(<<$\\,R/bits>>, Acc) -> escape_string(R, <<Acc/binary,$\\,$\\>>);
 escape_string(<<$",R/bits>>, Acc) -> escape_string(R, <<Acc/binary,$\\,$">>);
 escape_string(<<C,R/bits>>, Acc) -> escape_string(R, <<Acc/binary,C>>).
 
 params(Params) ->
-	maps:fold(fun
-		(Key, undefined, Acc) ->
-			[[$;, Key]|Acc];
-		(Key, Value, Acc) ->
-			[[$;, Key, $=, bare_item(Value)]|Acc]
-	end, [], Params).
+	[case Param of
+		{Key, true} -> [$;, Key];
+		{Key, Value} -> [$;, Key, $=, bare_item(Value)]
+	end || Param <- Params].
 
 -ifdef(TEST).
 struct_hd_identity_test_() ->
@@ -400,10 +500,12 @@ struct_hd_identity_test_() ->
 		Tests = jsx:decode(JSON, [return_maps]),
 		[
 			{iolist_to_binary(io_lib:format("~s: ~s", [filename:basename(File), Name])), fun() ->
+				io:format("expected json ~0p~n", [Expected0]),
 				Expected = expected_to_term(Expected0),
+				io:format("expected term: ~0p", [Expected]),
 				case HeaderType of
 					<<"dictionary">> ->
-						{Expected, _Order} = parse_dictionary(iolist_to_binary(dictionary(Expected)));
+						Expected = parse_dictionary(iolist_to_binary(dictionary(Expected)));
 					<<"item">> ->
 						Expected = parse_item(iolist_to_binary(item(Expected)));
 					<<"list">> ->