Merge pull request #197 from UmanShahzad/uman/configurable-json-codec

Allow configuring a json module with options for encode/decode in the json codec

README.md

@@ -79,7 +79,9 @@ connect(Opts) -> {ok, Connection :: epgsql:connection()} | {error, Reason :: epg
 
 connect(Host, Username, Password, Opts) -> {ok, C} | {error, Reason}.
 ```
+
 example:
+
 ```erlang
 {ok, C} = epgsql:connect("localhost", "username", "psss", #{
     database => "test_db",
@@ -146,7 +148,9 @@ Asynchronous connect example (applies to **epgsqli** too):
 %% @doc runs simple `SqlQuery' via given `Connection'
 squery(Connection, SqlQuery) -> ...
 ```
+
 examples:
+
 ```erlang
 epgsql:squery(C, "insert into account (name) values  ('alice'), ('bob')").
 > {ok,2}
@@ -237,6 +241,7 @@ end.
 {ok, Count, Columns, Rows} = epgsql:equery(C, "insert ... returning ...", [Parameters]).
 {error, Error}             = epgsql:equery(C, "invalid SQL", [Parameters]).
 ```
+
 `Parameters` - optional list of values to be bound to `$1`, `$2`, `$3`, etc.
 
 The extended query protocol combines parse, bind, and execute using
@@ -276,17 +281,20 @@ end.
 squery including final `{C, Ref, done}`.
 
 ### Prepared Query
+
 ```erlang
 {ok, Columns, Rows}        = epgsql:prepared_query(C, StatementName, [Parameters]).
 {ok, Count}                = epgsql:prepared_query(C, StatementName, [Parameters]).
 {ok, Count, Columns, Rows} = epgsql:prepared_query(C, StatementName, [Parameters]).
 {error, Error}             = epgsql:prepared_query(C, "non_existent_query", [Parameters]).
 ```
+
 `Parameters` - optional list of values to be bound to `$1`, `$2`, `$3`, etc.
 `StatementName` - name of query given with ```erlang epgsql:parse(C, StatementName, "select ...", []).```
 
 With prepared query one can parse a query giving it a name with `epgsql:parse` on start and reuse the name
 for all further queries with different parameters.
+
 ```erlang
 epgsql:parse(C, "inc", "select $1+1", []).
 epgsql:prepared_query(C, "inc", [4]).
@@ -322,11 +330,13 @@ squery including final `{C, Ref, done}`.
 For valid type names see `pgsql_types.erl`.
 
 `epgsqla:parse/2` sends `{C, Ref, {ok, Statement} | {error, Reason}}`.
+
 `epgsqli:parse/2` sends:
- - `{C, Ref, {types, Types}}`
- - `{C, Ref, {columns, Columns}}`
- - `{C, Ref, no_data}` if statement will not return rows
- - `{C, Ref, {error, Reason}}`
+
+- `{C, Ref, {types, Types}}`
+- `{C, Ref, {columns, Columns}}`
+- `{C, Ref, no_data}` if statement will not return rows
+- `{C, Ref, {error, Reason}}`
 
 ```erlang
 ok = epgsql:bind(C, Statement, [PortalName], ParameterValues).
@@ -351,6 +361,7 @@ both `epgsqla:bind/3` and `epgsqli:bind/3` send `{C, Ref, ok | {error, Reason}}`
 return value of `epgsql:execute/3`.
 
 `epgsqli:execute/3` sends
+
 - `{C, Ref, {data, Row}}`
 - `{C, Ref, {error, Reason}}`
 - `{C, Ref, suspended}` partial result was sent, more rows are available
@@ -367,7 +378,6 @@ All epgsql functions return `{error, Error}` when an error occurs.
 
 `epgsqla`/`epgsqli` modules' `close` and `sync` functions send `{C, Ref, ok}`.
 
-
 ### Batch execution
 
 Batch execution is `bind` + `execute` for several prepared statements.
@@ -389,7 +399,9 @@ example:
 ```
 
 `epgsqla:execute_batch/3` sends `{C, Ref, Results}`
+
 `epgsqli:execute_batch/3` sends
+
 - `{C, Ref, {data, Row}}`
 - `{C, Ref, {error, Reason}}`
 - `{C, Ref, {complete, {_Type, Count}}}`
@@ -422,7 +434,7 @@ PG type       | Representation
   point       |  `{10.2, 100.12}`
   int4range   | `[1,5)`
   hstore      | `{[ {binary(), binary() \| null} ]}`
-  json/jsonb  | `<<"{ \"key\": [ 1, 1.0, true, \"string\" ] }">>`
+  json/jsonb  | `<<"{ \"key\": [ 1, 1.0, true, \"string\" ] }">>` (see below for codec details)
   uuid        | `<<"123e4567-e89b-12d3-a456-426655440000">>`
   inet        | `inet:ip_address()`
   cidr        | `{ip_address(), Mask :: 0..32}`
@@ -432,14 +444,31 @@ PG type       | Representation
   tstzrange   | `{{Hour, Minute, Second.Microsecond}, {Hour, Minute, Second.Microsecond}}`
   daterange   | `{{Year, Month, Day}, {Year, Month, Day}}`
 
-  `timestamp` and `timestamptz` parameters can take `erlang:now()` format: `{MegaSeconds, Seconds, MicroSeconds}`
+`timestamp` and `timestamptz` parameters can take `erlang:now()` format: `{MegaSeconds, Seconds, MicroSeconds}`
+
+`int4range` is a range type for ints that obeys inclusive/exclusive semantics,
+bracket and parentheses respectively. Additionally, infinities are represented by the atoms `minus_infinity`
+and `plus_infinity`
+
+`tsrange`, `tstzrange`, `daterange` are range types for `timestamp`, `timestamptz` and `date`
+respectively. They can return `empty` atom as the result from a database if bounds are equal
+
+`json` and `jsonb` types can optionally use a custom JSON encoding/decoding module to accept
+and return erlang-formatted JSON. The module must implement the callbacks in `epgsql_codec_json`,
+which most popular open source JSON parsers will already, and you can specify it in the codec
+configuration like this:
+
+```erlang
+{epgsql_codec_json, JsonMod}
+
+% With options
+{epgsql_codec_json, JsonMod, EncodeOpts, DecodeOpts}
 
-  `int4range` is a range type for ints that obeys inclusive/exclusive semantics,
-  bracket and parentheses respectively. Additionally, infinities are represented by the atoms `minus_infinity`
-  and `plus_infinity`
+% Real world example using jiffy to return a map on decode
+{epgsql_codec_json, {jiffy, [], [return_maps]}}
+```
 
-  `tsrange`, `tstzrange`, `daterange` are range types for `timestamp`, `timestamptz` and `date`
-  respectively. They can return `empty` atom as the result from a database if bounds are equal
+Note that the decoded terms will be message-passed to the receiving process (i.e. copied), which may exhibit a performance hit if decoding large terms very frequently.
 
 ## Errors
 
@@ -482,6 +511,7 @@ Message formats:
 ```erlang
 {epgsql, Connection, {notification, Channel, Pid, Payload}}
 ```
+
 - `Connection`  - connection the notification occurred on
 - `Channel`  - channel the notification occurred on
 - `Pid`  - database session pid that sent notification
@@ -490,6 +520,7 @@ Message formats:
 ```erlang
 {epgsql, Connection, {notice, Error}}
 ```
+
 - `Connection`  - connection the notice occurred on
 - `Error`       - an `#error{}` record, see `epgsql.hrl`
 
@@ -510,7 +541,9 @@ Executes a function in a PostgreSQL transaction. It executes `BEGIN` prior to ex
 `ROLLBACK` if the function raises an exception and `COMMIT` if the function returns without an error.
 If it is successful, it returns the result of the function. The failure case may differ, depending on
 the options passed.
+
 Options (proplist or map):
+
 - `reraise` (default `true`): when set to true, the original exception will be re-thrown after rollback,
   otherwise `{rollback, ErrorReason}` will be returned
 - `ensure_committed` (default `false`): even when the callback returns without exception,
@@ -522,7 +555,6 @@ Options (proplist or map):
   appending them to `"BEGIN "` string. Eg `{begin_opts, "ISOLATION LEVEL SERIALIZABLE"}`.
   Beware of SQL injection! The value of `begin_opts` is not escaped!
 
-
 ### Command status
 
 `epgsql{a,i}:get_cmd_status(C) -> undefined | atom() | {atom(), integer()}`
@@ -571,10 +603,10 @@ See [pluggable_types.md](pluggable_types.md)
 epgsql is a community driven effort - we welcome contributions!
 Here's how to create a patch that's easy to integrate:
 
-* Create a new branch for the proposed fix.
-* Make sure it includes a test and documentation, if appropriate.
-* Open a pull request against the `devel` branch of epgsql.
-* Passing build in travis
+- Create a new branch for the proposed fix.
+- Make sure it includes a test and documentation, if appropriate.
+- Open a pull request against the `devel` branch of epgsql.
+- Passing build in travis
 
 ## Test Setup
 
@@ -582,11 +614,10 @@ In order to run the epgsql tests, you will need to install local
 Postgres database.
 
 NOTE: you will need the postgis and hstore extensions to run these
-tests!  On Ubuntu, you can install them with a command like this:
-
-1.  apt-get install postgresql-9.3-postgis-2.1 postgresql-contrib
+tests! On Ubuntu, you can install them with a command like this:
 
-2. `make test` # Runs the tests
+1. `apt-get install postgresql-9.3-postgis-2.1 postgresql-contrib`
+1. `make test` # Runs the tests
 
 NOTE 2: It's possible to run tests on exact postgres version by changing $PATH like
 

src/datatypes/epgsql_codec_json.erl

@@ -11,24 +11,54 @@
 
 -export([init/2, names/0, encode/3, decode/3, decode_text/3]).
 
--export_type([data/0]).
+-export_type([data/0, json_mod/0]).
 
 -type data() :: binary().
+-type json_mod()::module() | {module(), EncodeOpts::any(), DecodeOpts::any()}.
 
 -define(JSONB_VERSION_1, 1).
 
-%% TODO: JSON encode/decode `fun Mod:Name/1` / `{Mod, Name}` as option.
-%% Shall not pass `fun(_) -> .. end`, because of hot code upgrade problems.
-init(_, _) -> [].
+-optional_callbacks([encode/2, decode/2]).
+
+%% Encode erlang-formatted JSON, e.g. a map or a proplist,
+%% and return a valid JSON iolist or binary string.
+-callback encode(ErlJson::term()) -> EncodedJson::iodata().
+-callback encode(ErlJson::term(), EncodeOpts::term()) -> EncodedJson::iodata().
+
+%% Decode JSON binary string into erlang-formatted JSON.
+-callback decode(EncodedJson::binary()) -> ErlJson::term().
+-callback decode(EncodedJson::binary(), DecodeOpts::term()) -> ErlJson::term().
+
+%% JsonMod shall be a module that implements the callbacks defined by this module;
+%% encode/1, decode/1, and optionally the option-accepting variants.
+-spec init(JsonMod::json_mod(), epgsql_sock:pg_sock()) -> epgsql_codec:codec_state().
+init(JsonMod, _) ->
+    JsonMod.
 
 names() ->
     [json, jsonb].
 
+encode(ErlJson, json, JsonMod) when is_atom(JsonMod) ->
+    JsonMod:encode(ErlJson);
+encode(ErlJson, json, {JsonMod, EncodeOpts, _}) when is_atom(JsonMod) ->
+    JsonMod:encode(ErlJson, EncodeOpts);
+encode(ErlJson, jsonb, JsonMod) when is_atom(JsonMod) ->
+    [<<?JSONB_VERSION_1:8>> | JsonMod:encode(ErlJson)];
+encode(ErlJson, jsonb, {JsonMod, EncodeOpts, _}) when is_atom(JsonMod) ->
+    [<<?JSONB_VERSION_1:8>> | JsonMod:encode(ErlJson, EncodeOpts)];
 encode(Bin, json, _) ->
     Bin;
 encode(Bin, jsonb, _) ->
     [<<?JSONB_VERSION_1:8>> | Bin].
 
+decode(Bin, json, JsonMod) when is_atom(JsonMod) ->
+    JsonMod:decode(Bin);
+decode(Bin, json, {JsonMod, _, DecodeOpts}) when is_atom(JsonMod) ->
+    JsonMod:decode(Bin, DecodeOpts);
+decode(<<?JSONB_VERSION_1:8, Bin/binary>>, jsonb, JsonMod) when is_atom(JsonMod) ->
+    JsonMod:decode(Bin);
+decode(<<?JSONB_VERSION_1:8, Bin/binary>>, jsonb, {JsonMod, _, DecodeOpts}) when is_atom(JsonMod) ->
+    JsonMod:decode(Bin, DecodeOpts);
 decode(Bin, json, _) ->
     Bin;
 decode(<<?JSONB_VERSION_1:8, Bin/binary>>, jsonb, _) ->

test/epgsql_SUITE.erl

@@ -15,7 +15,7 @@
     end_per_suite/1
 ]).
 
--compile(export_all).
+-compile([export_all, nowarn_export_all]).
 
 modules() ->
     [
@@ -828,10 +828,24 @@ date_time_type(Config) ->
     end).
 
 json_type(Config) ->
-    check_type(Config, json, "'{}'", <<"{}">>,
-               [<<"{}">>, <<"[]">>, <<"1">>, <<"1.0">>, <<"true">>, <<"\"string\"">>, <<"{\"key\": []}">>]),
-    check_type(Config, jsonb, "'{}'", <<"{}">>,
-               [<<"{}">>, <<"[]">>, <<"1">>, <<"1.0">>, <<"true">>, <<"\"string\"">>, <<"{\"key\": []}">>]).
+    Module = ?config(module, Config),
+    epgsql_ct:with_connection(Config, fun(C) ->
+        check_type(Config, json, "'{}'", <<"{}">>,
+                [<<"{}">>, <<"[]">>, <<"1">>, <<"1.0">>, <<"true">>, <<"\"string\"">>, <<"{\"key\": []}">>],
+                get_type_col(json), C),
+        check_type(Config, jsonb, "'{}'", <<"{}">>,
+                [<<"{}">>, <<"[]">>, <<"1">>, <<"1.0">>, <<"true">>, <<"\"string\"">>, <<"{\"key\": []}">>],
+                get_type_col(jsonb), C),
+        epgsql:update_type_cache(C, [{epgsql_codec_json, epgsql_fake_json_mod}]),
+        RowId = "json_type_custom_mod_" ++ atom_to_list(Module),
+        {ok, 1} = Module:equery(C, "insert into test_table2 (c_varchar, c_json, c_jsonb) values ($1, $2, $3)", [RowId, {"{}"}, {"{}"}]),
+        Res = Module:equery(C, "select c_json, c_jsonb from test_table2 where c_varchar = $1", [RowId]),
+        ?assertMatch(
+            {ok, [#column{name = <<"c_json">>, type = json}, #column{name = <<"c_jsonb">>, type = jsonb}],
+                 [{{<<"{}">>}, {<<"{}">>}}]},
+            Res
+        )
+    end).
 
 misc_type(Config) ->
     check_type(Config, bool, "true", true, [true, false]),
@@ -1241,33 +1255,36 @@ with_transaction(Config) ->
 %% Internal functions
 %% ============================================================================
 
+get_type_col(Type) ->
+    "c_" ++ atom_to_list(Type).
+
 check_type(Config, Type, In, Out, Values) ->
-    Column = "c_" ++ atom_to_list(Type),
+    Column = get_type_col(Type),
     check_type(Config, Type, In, Out, Values, Column).
 
 check_type(Config, Type, In, Out, Values, Column) ->
-    Module = ?config(module, Config),
     epgsql_ct:with_connection(Config, fun(C) ->
-        Select = io_lib:format("select ~s::~w", [In, Type]),
-        Res = Module:equery(C, Select),
-        ?assertMatch({ok, [#column{type = Type}], [{Out}]}, Res),
-        Sql = io_lib:format("insert into test_table2 (~s) values ($1) returning ~s", [Column, Column]),
-        {ok, #statement{columns = [#column{type = Type}]} = S} = Module:parse(C, Sql),
-        Insert = fun(V) ->
-            ok = Module:bind(C, S, [V]),
-            {ok, 1, [{V2}]} = Module:execute(C, S),
-            case compare(Type, V, V2) of
-                true  -> ok;
-                false ->
-                    error({write_read_compare_failed,
-                           iolist_to_binary(
-                             io_lib:format("~p =/= ~p~n", [V, V2]))})
-            end,
-            ok = Module:sync(C)
-        end,
-        lists:foreach(Insert, [null, undefined | Values])
+        check_type(Config, Type, In, Out, Values, Column, C)
     end).
 
+check_type(Config, Type, In, Out, Values, Column, C) ->
+    Module = ?config(module, Config),
+    Select = io_lib:format("select ~s::~w", [In, Type]),
+    Res = Module:equery(C, Select),
+    ?assertMatch({ok, [#column{type = Type}], [{Out}]}, Res),
+    Sql = io_lib:format("insert into test_table2 (~s) values ($1) returning ~s", [Column, Column]),
+    {ok, #statement{columns = [#column{type = Type}]} = S} = Module:parse(C, Sql),
+    Insert = fun(V) ->
+        ok = Module:bind(C, S, [V]),
+        {ok, 1, [{V2}]} = Module:execute(C, S),
+        case compare(Type, V, V2) of
+            true  -> ok;
+            false -> error({write_read_compare_failed, io_lib:format("~p =/= ~p", [V, V2])})
+        end,
+        ok = Module:sync(C)
+    end,
+    lists:foreach(Insert, [null, undefined | Values]).
+
 compare(_Type, null, null)      -> true;
 compare(_Type, undefined, null) -> true;
 compare(float4, V1, V2) when is_float(V1) -> abs(V2 - V1) < 0.000001;

test/epgsql_fake_json_mod.erl

@@ -0,0 +1,16 @@
+-module(epgsql_fake_json_mod).
+
+-export([encode/1]).
+-export([decode/1]).
+
+encode(ErlJson) ->
+    encode(ErlJson, []).
+
+encode({ErlJson}, _EncodeOpts) ->
+    ErlJson.
+
+decode(EncodedJson) ->
+    decode(EncodedJson, []).
+
+decode(EncodedJson, _DecodeOpts) ->
+    {EncodedJson}.