Transforms for jsonb to PL/Python
Add a new contrib module jsonb_plpython that provide a transform between jsonb and PL/Python. jsonb values are converted to appropriate Python types such as dicts and lists, and vice versa. Author: Anthony Bykov <a.bykov@postgrespro.ru> Reviewed-by: Aleksander Alekseev <a.alekseev@postgrespro.ru> Reviewed-by: Nikita Glukhov <n.gluhov@postgrespro.ru>
This commit is contained in:
Peter Eisentraut
parent
a437551a22
commit
3f44e3db72
|
|
@ -81,9 +81,9 @@ ALWAYS_SUBDIRS += hstore_plperl
|
|||
endif
|
||||
|
||||
ifeq ($(with_python),yes)
|
||||
SUBDIRS += hstore_plpython ltree_plpython
|
||||
SUBDIRS += hstore_plpython jsonb_plpython ltree_plpython
|
||||
else
|
||||
ALWAYS_SUBDIRS += hstore_plpython ltree_plpython
|
||||
ALWAYS_SUBDIRS += hstore_plpython jsonb_plpython ltree_plpython
|
||||
endif
|
||||
|
||||
# Missing:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
# Generated subdirectories
|
||||
/expected/python3/
|
||||
/log/
|
||||
/results/
|
||||
/sql/python3/
|
||||
/tmp_check/
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# contrib/jsonb_plpython/Makefile
|
||||
|
||||
MODULE_big = jsonb_plpython$(python_majorversion)
|
||||
OBJS = jsonb_plpython.o $(WIN32RES)
|
||||
PGFILEDESC = "jsonb_plpython - transform between jsonb and plpythonu"
|
||||
|
||||
PG_CPPFLAGS = -I$(top_srcdir)/src/pl/plpython $(python_includespec) -DPLPYTHON_LIBNAME='"plpython$(python_majorversion)"'
|
||||
|
||||
EXTENSION = jsonb_plpythonu jsonb_plpython2u jsonb_plpython3u
|
||||
DATA = jsonb_plpythonu--1.0.sql jsonb_plpython2u--1.0.sql jsonb_plpython3u--1.0.sql
|
||||
|
||||
REGRESS = jsonb_plpython
|
||||
REGRESS_PLPYTHON3_MANGLE := $(REGRESS)
|
||||
|
||||
ifdef USE_PGXS
|
||||
PG_CONFIG = pg_config
|
||||
PGXS := $(shell $(PG_CONFIG) --pgxs)
|
||||
include $(PGXS)
|
||||
else
|
||||
subdir = contrib/jsonb_plpython
|
||||
top_builddir = ../..
|
||||
include $(top_builddir)/src/Makefile.global
|
||||
include $(top_srcdir)/contrib/contrib-global.mk
|
||||
endif
|
||||
|
||||
# We must link libpython explicitly
|
||||
ifeq ($(PORTNAME), win32)
|
||||
# ... see silliness in plpython Makefile ...
|
||||
SHLIB_LINK += $(sort $(wildcard ../../src/pl/plpython/libpython*.a))
|
||||
else
|
||||
rpathdir = $(python_libdir)
|
||||
SHLIB_LINK += $(python_libspec) $(python_additional_libs)
|
||||
endif
|
||||
|
||||
ifeq ($(python_majorversion),2)
|
||||
REGRESS_OPTS += --load-extension=plpythonu --load-extension=jsonb_plpythonu
|
||||
endif
|
||||
|
||||
include $(top_srcdir)/src/pl/plpython/regress-python3-mangle.mk
|
||||
|
|
@ -0,0 +1,347 @@
|
|||
CREATE EXTENSION jsonb_plpython2u CASCADE;
|
||||
NOTICE: installing required extension "plpython2u"
|
||||
-- test jsonb -> python dict
|
||||
CREATE FUNCTION test1(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, dict)
|
||||
plpy.info(sorted(val.items()))
|
||||
return len(val)
|
||||
$$;
|
||||
SELECT test1('{"a": 1, "c": "NULL"}'::jsonb);
|
||||
INFO: [('a', Decimal('1')), ('c', 'NULL')]
|
||||
test1
|
||||
-------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
-- test jsonb -> python dict
|
||||
-- complex dict with dicts as value
|
||||
CREATE FUNCTION test1complex(val jsonb) RETURNS int
|
||||
LANGUAGE plpython2u
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, dict)
|
||||
assert(val == {"d": {"d": 1}})
|
||||
return len(val)
|
||||
$$;
|
||||
SELECT test1complex('{"d": {"d": 1}}'::jsonb);
|
||||
test1complex
|
||||
--------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- test jsonb[] -> python dict
|
||||
-- dict with array as value
|
||||
CREATE FUNCTION test1arr(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, dict)
|
||||
assert(val == {"d": [12, 1]})
|
||||
return len(val)
|
||||
$$;
|
||||
SELECT test1arr('{"d":[12, 1]}'::jsonb);
|
||||
test1arr
|
||||
----------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- test jsonb[] -> python list
|
||||
-- simple list
|
||||
CREATE FUNCTION test2arr(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, list)
|
||||
assert(val == [12, 1])
|
||||
return len(val)
|
||||
$$;
|
||||
SELECT test2arr('[12, 1]'::jsonb);
|
||||
test2arr
|
||||
----------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
-- test jsonb[] -> python list
|
||||
-- array of dicts
|
||||
CREATE FUNCTION test3arr(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, list)
|
||||
assert(val == [{"a": 1,"b": 2}, {"c": 3,"d": 4}])
|
||||
return len(val)
|
||||
$$;
|
||||
SELECT test3arr('[{"a": 1, "b": 2}, {"c": 3,"d": 4}]'::jsonb);
|
||||
test3arr
|
||||
----------
|
||||
2
|
||||
(1 row)
|
||||
|
||||
-- test jsonb int -> python int
|
||||
CREATE FUNCTION test1int(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert(val == 1)
|
||||
return val
|
||||
$$;
|
||||
SELECT test1int('1'::jsonb);
|
||||
test1int
|
||||
----------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- test jsonb string -> python string
|
||||
CREATE FUNCTION test1string(val jsonb) RETURNS text
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert(val == "a")
|
||||
return val
|
||||
$$;
|
||||
SELECT test1string('"a"'::jsonb);
|
||||
test1string
|
||||
-------------
|
||||
a
|
||||
(1 row)
|
||||
|
||||
-- test jsonb null -> python None
|
||||
CREATE FUNCTION test1null(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert(val == None)
|
||||
return 1
|
||||
$$;
|
||||
SELECT test1null('null'::jsonb);
|
||||
test1null
|
||||
-----------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- test python -> jsonb
|
||||
CREATE FUNCTION roundtrip(val jsonb) RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
as $$
|
||||
return val
|
||||
$$;
|
||||
SELECT roundtrip('null'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('1'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('1234567890.0987654321'::jsonb);
|
||||
roundtrip
|
||||
-----------------------
|
||||
1234567890.0987654321
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('-1234567890.0987654321'::jsonb);
|
||||
roundtrip
|
||||
------------------------
|
||||
-1234567890.0987654321
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('true'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
true
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('"string"'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
"string"
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('{"1": null}'::jsonb);
|
||||
roundtrip
|
||||
-------------
|
||||
{"1": null}
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('{"1": 1}'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
{"1": 1}
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('{"1": true}'::jsonb);
|
||||
roundtrip
|
||||
-------------
|
||||
{"1": true}
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('{"1": "string"}'::jsonb);
|
||||
roundtrip
|
||||
-----------------
|
||||
{"1": "string"}
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('[null]'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
[null]
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('[1]'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
[1]
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('[true]'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
[true]
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('["string"]'::jsonb);
|
||||
roundtrip
|
||||
------------
|
||||
["string"]
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('[null, 1]'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
[null, 1]
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('[1, true]'::jsonb);
|
||||
roundtrip
|
||||
-----------
|
||||
[1, true]
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('[true, "string"]'::jsonb);
|
||||
roundtrip
|
||||
------------------
|
||||
[true, "string"]
|
||||
(1 row)
|
||||
|
||||
SELECT roundtrip('["string", "string2"]'::jsonb);
|
||||
roundtrip
|
||||
-----------------------
|
||||
["string", "string2"]
|
||||
(1 row)
|
||||
|
||||
-- test python infinity -> jsonb
|
||||
CREATE FUNCTION test1inf() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = float('inf')
|
||||
print(x)
|
||||
return x
|
||||
$$;
|
||||
SELECT test1inf();
|
||||
ERROR: could not convert value "inf" to jsonb
|
||||
CONTEXT: while creating return value
|
||||
PL/Python function "test1inf"
|
||||
-- test python -infinity -> jsonb
|
||||
CREATE FUNCTION test2inf() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = float('-inf')
|
||||
print(x)
|
||||
return x
|
||||
$$;
|
||||
SELECT test2inf();
|
||||
ERROR: could not convert value "-inf" to jsonb
|
||||
CONTEXT: while creating return value
|
||||
PL/Python function "test2inf"
|
||||
-- test python NaN -> jsonb
|
||||
CREATE FUNCTION test1nan() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = float('nan')
|
||||
return x
|
||||
$$;
|
||||
SELECT test1nan();
|
||||
test1nan
|
||||
----------
|
||||
NaN
|
||||
(1 row)
|
||||
|
||||
-- complex numbers -> jsonb
|
||||
CREATE FUNCTION testComplexNumbers() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = 1 + 2j
|
||||
return x
|
||||
$$;
|
||||
SELECT testComplexNumbers();
|
||||
ERROR: could not convert value "(1+2j)" to jsonb
|
||||
CONTEXT: while creating return value
|
||||
PL/Python function "testcomplexnumbers"
|
||||
-- range -> jsonb
|
||||
CREATE FUNCTION testRange() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = range(3)
|
||||
return x
|
||||
$$;
|
||||
SELECT testRange();
|
||||
testrange
|
||||
-----------
|
||||
[0, 1, 2]
|
||||
(1 row)
|
||||
|
||||
-- 0xff -> jsonb
|
||||
CREATE FUNCTION testDecimal() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = 0xff
|
||||
return x
|
||||
$$;
|
||||
SELECT testDecimal();
|
||||
testdecimal
|
||||
-------------
|
||||
255
|
||||
(1 row)
|
||||
|
||||
-- tuple -> jsonb
|
||||
CREATE FUNCTION testTuple() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = (1, 'String', None)
|
||||
return x
|
||||
$$;
|
||||
SELECT testTuple();
|
||||
testtuple
|
||||
---------------------
|
||||
[1, "String", null]
|
||||
(1 row)
|
||||
|
||||
-- interesting dict -> jsonb
|
||||
CREATE FUNCTION test_dict1() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = {"a": 1, None: 2, 33: 3}
|
||||
return x
|
||||
$$;
|
||||
SELECT test_dict1();
|
||||
test_dict1
|
||||
--------------------------
|
||||
{"": 2, "a": 1, "33": 3}
|
||||
(1 row)
|
||||
|
||||
|
|
@ -0,0 +1,453 @@
|
|||
#include "postgres.h"
|
||||
|
||||
#include "plpython.h"
|
||||
#include "plpy_elog.h"
|
||||
#include "plpy_typeio.h"
|
||||
#include "utils/jsonb.h"
|
||||
#include "utils/fmgrprotos.h"
|
||||
|
||||
PG_MODULE_MAGIC;
|
||||
|
||||
void _PG_init(void);
|
||||
|
||||
/* for PLyObject_AsString in plpy_typeio.c */
|
||||
typedef char *(*PLyObject_AsString_t) (PyObject *plrv);
|
||||
static PLyObject_AsString_t PLyObject_AsString_p;
|
||||
|
||||
typedef void (*PLy_elog_impl_t) (int elevel, const char *fmt,...);
|
||||
static PLy_elog_impl_t PLy_elog_impl_p;
|
||||
|
||||
/*
|
||||
* decimal_constructor is a function from python library and used
|
||||
* for transforming strings into python decimal type
|
||||
*/
|
||||
static PyObject *decimal_constructor;
|
||||
|
||||
static PyObject *PLyObject_FromJsonbContainer(JsonbContainer *jsonb);
|
||||
static JsonbValue *PLyObject_ToJsonbValue(PyObject *obj,
|
||||
JsonbParseState **jsonb_state, bool is_elem);
|
||||
|
||||
#if PY_MAJOR_VERSION >= 3
|
||||
typedef PyObject *(*PLyUnicode_FromStringAndSize_t)
|
||||
(const char *s, Py_ssize_t size);
|
||||
static PLyUnicode_FromStringAndSize_t PLyUnicode_FromStringAndSize_p;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Module initialize function: fetch function pointers for cross-module calls.
|
||||
*/
|
||||
void
|
||||
_PG_init(void)
|
||||
{
|
||||
/* Asserts verify that typedefs above match original declarations */
|
||||
AssertVariableIsOfType(&PLyObject_AsString, PLyObject_AsString_t);
|
||||
PLyObject_AsString_p = (PLyObject_AsString_t)
|
||||
load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyObject_AsString",
|
||||
true, NULL);
|
||||
#if PY_MAJOR_VERSION >= 3
|
||||
AssertVariableIsOfType(&PLyUnicode_FromStringAndSize, PLyUnicode_FromStringAndSize_t);
|
||||
PLyUnicode_FromStringAndSize_p = (PLyUnicode_FromStringAndSize_t)
|
||||
load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLyUnicode_FromStringAndSize",
|
||||
true, NULL);
|
||||
#endif
|
||||
|
||||
AssertVariableIsOfType(&PLy_elog_impl, PLy_elog_impl_t);
|
||||
PLy_elog_impl_p = (PLy_elog_impl_t)
|
||||
load_external_function("$libdir/" PLPYTHON_LIBNAME, "PLy_elog_impl",
|
||||
true, NULL);
|
||||
}
|
||||
|
||||
/* These defines must be after the _PG_init */
|
||||
#define PLyObject_AsString (PLyObject_AsString_p)
|
||||
#define PLyUnicode_FromStringAndSize (PLyUnicode_FromStringAndSize_p)
|
||||
#undef PLy_elog
|
||||
#define PLy_elog (PLy_elog_impl_p)
|
||||
|
||||
/*
|
||||
* PLyString_FromJsonbValue
|
||||
*
|
||||
* Transform string JsonbValue to Python string.
|
||||
*/
|
||||
static PyObject *
|
||||
PLyString_FromJsonbValue(JsonbValue *jbv)
|
||||
{
|
||||
Assert(jbv->type == jbvString);
|
||||
|
||||
return PyString_FromStringAndSize(jbv->val.string.val, jbv->val.string.len);
|
||||
}
|
||||
|
||||
/*
|
||||
* PLyString_ToJsonbValue
|
||||
*
|
||||
* Transform Python string to JsonbValue.
|
||||
*/
|
||||
static void
|
||||
PLyString_ToJsonbValue(PyObject *obj, JsonbValue *jbvElem)
|
||||
{
|
||||
jbvElem->type = jbvString;
|
||||
jbvElem->val.string.val = PLyObject_AsString(obj);
|
||||
jbvElem->val.string.len = strlen(jbvElem->val.string.val);
|
||||
}
|
||||
|
||||
/*
|
||||
* PLyObject_FromJsonbValue
|
||||
*
|
||||
* Transform JsonbValue to PyObject.
|
||||
*/
|
||||
static PyObject *
|
||||
PLyObject_FromJsonbValue(JsonbValue *jsonbValue)
|
||||
{
|
||||
switch (jsonbValue->type)
|
||||
{
|
||||
case jbvNull:
|
||||
Py_RETURN_NONE;
|
||||
|
||||
case jbvBinary:
|
||||
return PLyObject_FromJsonbContainer(jsonbValue->val.binary.data);
|
||||
|
||||
case jbvNumeric:
|
||||
{
|
||||
Datum num;
|
||||
char *str;
|
||||
|
||||
num = NumericGetDatum(jsonbValue->val.numeric);
|
||||
str = DatumGetCString(DirectFunctionCall1(numeric_out, num));
|
||||
|
||||
return PyObject_CallFunction(decimal_constructor, "s", str);
|
||||
}
|
||||
|
||||
case jbvString:
|
||||
return PLyString_FromJsonbValue(jsonbValue);
|
||||
|
||||
case jbvBool:
|
||||
if (jsonbValue->val.boolean)
|
||||
Py_RETURN_TRUE;
|
||||
else
|
||||
Py_RETURN_FALSE;
|
||||
|
||||
default:
|
||||
elog(ERROR, "unexpected jsonb value type: %d", jsonbValue->type);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* PLyObject_FromJsonb
|
||||
*
|
||||
* Transform JsonbContainer to PyObject.
|
||||
*/
|
||||
static PyObject *
|
||||
PLyObject_FromJsonbContainer(JsonbContainer *jsonb)
|
||||
{
|
||||
JsonbIteratorToken r;
|
||||
JsonbValue v;
|
||||
JsonbIterator *it;
|
||||
PyObject *result;
|
||||
|
||||
it = JsonbIteratorInit(jsonb);
|
||||
r = JsonbIteratorNext(&it, &v, true);
|
||||
|
||||
switch (r)
|
||||
{
|
||||
case WJB_BEGIN_ARRAY:
|
||||
if (v.val.array.rawScalar)
|
||||
{
|
||||
JsonbValue tmp;
|
||||
|
||||
if ((r = JsonbIteratorNext(&it, &v, true)) != WJB_ELEM ||
|
||||
(r = JsonbIteratorNext(&it, &tmp, true)) != WJB_END_ARRAY ||
|
||||
(r = JsonbIteratorNext(&it, &tmp, true)) != WJB_DONE)
|
||||
elog(ERROR, "unexpected jsonb token: %d", r);
|
||||
|
||||
result = PLyObject_FromJsonbValue(&v);
|
||||
}
|
||||
else
|
||||
{
|
||||
/* array in v */
|
||||
result = PyList_New(0);
|
||||
if (!result)
|
||||
return NULL;
|
||||
|
||||
while ((r = JsonbIteratorNext(&it, &v, true)) != WJB_DONE)
|
||||
{
|
||||
if (r == WJB_ELEM)
|
||||
{
|
||||
PyObject *elem = PLyObject_FromJsonbValue(&v);
|
||||
|
||||
PyList_Append(result, elem);
|
||||
Py_XDECREF(elem);
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case WJB_BEGIN_OBJECT:
|
||||
result = PyDict_New();
|
||||
if (!result)
|
||||
return NULL;
|
||||
|
||||
while ((r = JsonbIteratorNext(&it, &v, true)) != WJB_DONE)
|
||||
{
|
||||
if (r == WJB_KEY)
|
||||
{
|
||||
PyObject *key = PLyString_FromJsonbValue(&v);
|
||||
|
||||
if (!key)
|
||||
return NULL;
|
||||
|
||||
r = JsonbIteratorNext(&it, &v, true);
|
||||
|
||||
if (r == WJB_VALUE)
|
||||
{
|
||||
PyObject *value = PLyObject_FromJsonbValue(&v);
|
||||
|
||||
if (!value)
|
||||
{
|
||||
Py_XDECREF(key);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
PyDict_SetItem(result, key, value);
|
||||
Py_XDECREF(value);
|
||||
}
|
||||
|
||||
Py_XDECREF(key);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
elog(ERROR, "unexpected jsonb token: %d", r);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* PLyMapping_ToJsonbValue
|
||||
*
|
||||
* Transform Python dict to JsonbValue.
|
||||
*/
|
||||
static JsonbValue *
|
||||
PLyMapping_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state)
|
||||
{
|
||||
Py_ssize_t pcount;
|
||||
JsonbValue *out = NULL;
|
||||
|
||||
/* We need it volatile, since we use it after longjmp */
|
||||
volatile PyObject *items_v = NULL;
|
||||
|
||||
pcount = PyMapping_Size(obj);
|
||||
items_v = PyMapping_Items(obj);
|
||||
|
||||
PG_TRY();
|
||||
{
|
||||
Py_ssize_t i;
|
||||
PyObject *items;
|
||||
|
||||
items = (PyObject *) items_v;
|
||||
|
||||
pushJsonbValue(jsonb_state, WJB_BEGIN_OBJECT, NULL);
|
||||
|
||||
for (i = 0; i < pcount; i++)
|
||||
{
|
||||
JsonbValue jbvKey;
|
||||
PyObject *item = PyList_GetItem(items, i);
|
||||
PyObject *key = PyTuple_GetItem(item, 0);
|
||||
PyObject *value = PyTuple_GetItem(item, 1);
|
||||
|
||||
/* Python dictionary can have None as key */
|
||||
if (key == Py_None)
|
||||
{
|
||||
jbvKey.type = jbvString;
|
||||
jbvKey.val.string.len = 0;
|
||||
jbvKey.val.string.val = "";
|
||||
}
|
||||
else
|
||||
{
|
||||
/* All others types of keys we serialize to string */
|
||||
PLyString_ToJsonbValue(key, &jbvKey);
|
||||
}
|
||||
|
||||
(void) pushJsonbValue(jsonb_state, WJB_KEY, &jbvKey);
|
||||
(void) PLyObject_ToJsonbValue(value, jsonb_state, false);
|
||||
}
|
||||
|
||||
out = pushJsonbValue(jsonb_state, WJB_END_OBJECT, NULL);
|
||||
}
|
||||
PG_CATCH();
|
||||
{
|
||||
Py_DECREF(items_v);
|
||||
PG_RE_THROW();
|
||||
}
|
||||
PG_END_TRY();
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/*
|
||||
* PLySequence_ToJsonbValue
|
||||
*
|
||||
* Transform python list to JsonbValue. Expects transformed PyObject and
|
||||
* a state required for jsonb construction.
|
||||
*/
|
||||
static JsonbValue *
|
||||
PLySequence_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state)
|
||||
{
|
||||
Py_ssize_t i;
|
||||
Py_ssize_t pcount;
|
||||
|
||||
pcount = PySequence_Size(obj);
|
||||
|
||||
pushJsonbValue(jsonb_state, WJB_BEGIN_ARRAY, NULL);
|
||||
|
||||
for (i = 0; i < pcount; i++)
|
||||
{
|
||||
PyObject *value = PySequence_GetItem(obj, i);
|
||||
|
||||
(void) PLyObject_ToJsonbValue(value, jsonb_state, true);
|
||||
}
|
||||
|
||||
return pushJsonbValue(jsonb_state, WJB_END_ARRAY, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
* PLyNumber_ToJsonbValue(PyObject *obj)
|
||||
*
|
||||
* Transform python number to JsonbValue.
|
||||
*/
|
||||
static JsonbValue *
|
||||
PLyNumber_ToJsonbValue(PyObject *obj, JsonbValue *jbvNum)
|
||||
{
|
||||
Numeric num;
|
||||
char *str = PLyObject_AsString(obj);
|
||||
|
||||
PG_TRY();
|
||||
{
|
||||
num = DatumGetNumeric(DirectFunctionCall3(numeric_in,
|
||||
CStringGetDatum(str), 0, -1));
|
||||
}
|
||||
PG_CATCH();
|
||||
{
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_DATATYPE_MISMATCH),
|
||||
(errmsg("could not convert value \"%s\" to jsonb", str))));
|
||||
}
|
||||
PG_END_TRY();
|
||||
|
||||
pfree(str);
|
||||
|
||||
jbvNum->type = jbvNumeric;
|
||||
jbvNum->val.numeric = num;
|
||||
|
||||
return jbvNum;
|
||||
}
|
||||
|
||||
/*
|
||||
* PLyObject_ToJsonbValue(PyObject *obj)
|
||||
*
|
||||
* Transform python object to JsonbValue.
|
||||
*/
|
||||
static JsonbValue *
|
||||
PLyObject_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state, bool is_elem)
|
||||
{
|
||||
JsonbValue buf;
|
||||
JsonbValue *out;
|
||||
|
||||
if (!(PyString_Check(obj) || PyUnicode_Check(obj)))
|
||||
{
|
||||
if (PySequence_Check(obj))
|
||||
return PLySequence_ToJsonbValue(obj, jsonb_state);
|
||||
else if (PyMapping_Check(obj))
|
||||
return PLyMapping_ToJsonbValue(obj, jsonb_state);
|
||||
}
|
||||
|
||||
/* Allocate JsonbValue in heap only if it is raw scalar value. */
|
||||
if (*jsonb_state)
|
||||
out = &buf;
|
||||
else
|
||||
out = palloc(sizeof(JsonbValue));
|
||||
|
||||
if (obj == Py_None)
|
||||
out->type = jbvNull;
|
||||
else if (PyString_Check(obj) || PyUnicode_Check(obj))
|
||||
PLyString_ToJsonbValue(obj, out);
|
||||
/*
|
||||
* PyNumber_Check() returns true for booleans, so boolean check should come
|
||||
* first.
|
||||
*/
|
||||
else if (PyBool_Check(obj))
|
||||
{
|
||||
out = palloc(sizeof(JsonbValue));
|
||||
out->type = jbvBool;
|
||||
out->val.boolean = (obj == Py_True);
|
||||
}
|
||||
else if (PyNumber_Check(obj))
|
||||
out = PLyNumber_ToJsonbValue(obj, out);
|
||||
else
|
||||
ereport(ERROR,
|
||||
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
||||
(errmsg("Python type \"%s\" cannot be transformed to jsonb",
|
||||
PLyObject_AsString((PyObject *) obj->ob_type)))));
|
||||
|
||||
/* Push result into 'jsonb_state' unless it is raw scalar value. */
|
||||
return (*jsonb_state ?
|
||||
pushJsonbValue(jsonb_state, is_elem ? WJB_ELEM : WJB_VALUE, out) :
|
||||
out);
|
||||
}
|
||||
|
||||
/*
|
||||
* plpython_to_jsonb
|
||||
*
|
||||
* Transform python object to Jsonb datum
|
||||
*/
|
||||
PG_FUNCTION_INFO_V1(plpython_to_jsonb);
|
||||
Datum
|
||||
plpython_to_jsonb(PG_FUNCTION_ARGS)
|
||||
{
|
||||
PyObject *obj;
|
||||
JsonbValue *out;
|
||||
JsonbParseState *jsonb_state = NULL;
|
||||
|
||||
obj = (PyObject *) PG_GETARG_POINTER(0);
|
||||
out = PLyObject_ToJsonbValue(obj, &jsonb_state, true);
|
||||
PG_RETURN_POINTER(JsonbValueToJsonb(out));
|
||||
}
|
||||
|
||||
/*
|
||||
* jsonb_to_plpython
|
||||
*
|
||||
* Transform Jsonb datum to PyObject and return it as internal.
|
||||
*/
|
||||
PG_FUNCTION_INFO_V1(jsonb_to_plpython);
|
||||
Datum
|
||||
jsonb_to_plpython(PG_FUNCTION_ARGS)
|
||||
{
|
||||
PyObject *result;
|
||||
Jsonb *in = PG_GETARG_JSONB_P(0);
|
||||
|
||||
/*
|
||||
* Initialize pointer to Decimal constructor. First we try "cdecimal", C
|
||||
* version of decimal library. In case of failure we use slower "decimal"
|
||||
* module.
|
||||
*/
|
||||
if (!decimal_constructor)
|
||||
{
|
||||
PyObject *decimal_module = PyImport_ImportModule("cdecimal");
|
||||
|
||||
if (!decimal_module)
|
||||
{
|
||||
PyErr_Clear();
|
||||
decimal_module = PyImport_ImportModule("decimal");
|
||||
}
|
||||
Assert(decimal_module);
|
||||
decimal_constructor = PyObject_GetAttrString(decimal_module, "Decimal");
|
||||
}
|
||||
|
||||
result = PLyObject_FromJsonbContainer(&in->root);
|
||||
if (!result)
|
||||
PLy_elog(ERROR, "transformation from jsonb to Python failed");
|
||||
|
||||
return PointerGetDatum(result);
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
/* contrib/jsonb_plpython/jsonb_plpython2u--1.0.sql */
|
||||
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "CREATE EXTENSION jsonb_plpython2u" to load this file. \quit
|
||||
|
||||
CREATE FUNCTION jsonb_to_plpython2(val internal) RETURNS internal
|
||||
LANGUAGE C STRICT IMMUTABLE
|
||||
AS 'MODULE_PATHNAME', 'jsonb_to_plpython';
|
||||
|
||||
CREATE FUNCTION plpython2_to_jsonb(val internal) RETURNS jsonb
|
||||
LANGUAGE C STRICT IMMUTABLE
|
||||
AS 'MODULE_PATHNAME', 'plpython_to_jsonb';
|
||||
|
||||
CREATE TRANSFORM FOR jsonb LANGUAGE plpython2u (
|
||||
FROM SQL WITH FUNCTION jsonb_to_plpython2(internal),
|
||||
TO SQL WITH FUNCTION plpython2_to_jsonb(internal)
|
||||
);
|
||||
|
||||
COMMENT ON TRANSFORM FOR jsonb LANGUAGE plpython2u IS 'transform between jsonb and Python';
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# jsonb_plpython2u extension
|
||||
comment = 'transform between jsonb and plpython2u'
|
||||
default_version = '1.0'
|
||||
module_pathname = '$libdir/jsonb_plpython2'
|
||||
relocatable = true
|
||||
requires = 'plpython2u'
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
/* contrib/jsonb_plpython/jsonb_plpython3u--1.0.sql */
|
||||
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "CREATE EXTENSION jsonb_plpython3u" to load this file. \quit
|
||||
|
||||
CREATE FUNCTION jsonb_to_plpython3(val internal) RETURNS internal
|
||||
LANGUAGE C STRICT IMMUTABLE
|
||||
AS 'MODULE_PATHNAME', 'jsonb_to_plpython';
|
||||
|
||||
CREATE FUNCTION plpython3_to_jsonb(val internal) RETURNS jsonb
|
||||
LANGUAGE C STRICT IMMUTABLE
|
||||
AS 'MODULE_PATHNAME', 'plpython_to_jsonb';
|
||||
|
||||
CREATE TRANSFORM FOR jsonb LANGUAGE plpython3u (
|
||||
FROM SQL WITH FUNCTION jsonb_to_plpython3(internal),
|
||||
TO SQL WITH FUNCTION plpython3_to_jsonb(internal)
|
||||
);
|
||||
|
||||
COMMENT ON TRANSFORM FOR jsonb LANGUAGE plpython3u IS 'transform between jsonb and Python';
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# jsonb_plpython3u extension
|
||||
comment = 'transform between jsonb and plpython3u'
|
||||
default_version = '1.0'
|
||||
module_pathname = '$libdir/jsonb_plpython3'
|
||||
relocatable = true
|
||||
requires = 'plpython3u'
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
/* contrib/jsonb_plpython/jsonb_plpythonu--1.0.sql */
|
||||
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "CREATE EXTENSION jsonb_plpythonu" to load this file. \quit
|
||||
|
||||
CREATE FUNCTION jsonb_to_plpython(val internal) RETURNS internal
|
||||
LANGUAGE C STRICT IMMUTABLE
|
||||
AS 'MODULE_PATHNAME';
|
||||
|
||||
CREATE FUNCTION plpython_to_jsonb(val internal) RETURNS jsonb
|
||||
LANGUAGE C STRICT IMMUTABLE
|
||||
AS 'MODULE_PATHNAME';
|
||||
|
||||
CREATE TRANSFORM FOR jsonb LANGUAGE plpythonu (
|
||||
FROM SQL WITH FUNCTION jsonb_to_plpython(internal),
|
||||
TO SQL WITH FUNCTION plpython_to_jsonb(internal)
|
||||
);
|
||||
|
||||
COMMENT ON TRANSFORM FOR jsonb LANGUAGE plpythonu IS 'transform between jsonb and Python';
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# jsonb_plpythonu extension
|
||||
comment = 'transform between jsonb and plpythonu'
|
||||
default_version = '1.0'
|
||||
module_pathname = '$libdir/jsonb_plpython2'
|
||||
relocatable = true
|
||||
requires = 'plpythonu'
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
CREATE EXTENSION jsonb_plpython2u CASCADE;
|
||||
|
||||
-- test jsonb -> python dict
|
||||
CREATE FUNCTION test1(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, dict)
|
||||
plpy.info(sorted(val.items()))
|
||||
return len(val)
|
||||
$$;
|
||||
|
||||
SELECT test1('{"a": 1, "c": "NULL"}'::jsonb);
|
||||
|
||||
-- test jsonb -> python dict
|
||||
-- complex dict with dicts as value
|
||||
CREATE FUNCTION test1complex(val jsonb) RETURNS int
|
||||
LANGUAGE plpython2u
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, dict)
|
||||
assert(val == {"d": {"d": 1}})
|
||||
return len(val)
|
||||
$$;
|
||||
|
||||
SELECT test1complex('{"d": {"d": 1}}'::jsonb);
|
||||
|
||||
|
||||
-- test jsonb[] -> python dict
|
||||
-- dict with array as value
|
||||
CREATE FUNCTION test1arr(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, dict)
|
||||
assert(val == {"d": [12, 1]})
|
||||
return len(val)
|
||||
$$;
|
||||
|
||||
SELECT test1arr('{"d":[12, 1]}'::jsonb);
|
||||
|
||||
-- test jsonb[] -> python list
|
||||
-- simple list
|
||||
CREATE FUNCTION test2arr(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, list)
|
||||
assert(val == [12, 1])
|
||||
return len(val)
|
||||
$$;
|
||||
|
||||
SELECT test2arr('[12, 1]'::jsonb);
|
||||
|
||||
-- test jsonb[] -> python list
|
||||
-- array of dicts
|
||||
CREATE FUNCTION test3arr(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert isinstance(val, list)
|
||||
assert(val == [{"a": 1,"b": 2}, {"c": 3,"d": 4}])
|
||||
return len(val)
|
||||
$$;
|
||||
|
||||
SELECT test3arr('[{"a": 1, "b": 2}, {"c": 3,"d": 4}]'::jsonb);
|
||||
|
||||
-- test jsonb int -> python int
|
||||
CREATE FUNCTION test1int(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert(val == 1)
|
||||
return val
|
||||
$$;
|
||||
|
||||
SELECT test1int('1'::jsonb);
|
||||
|
||||
-- test jsonb string -> python string
|
||||
CREATE FUNCTION test1string(val jsonb) RETURNS text
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert(val == "a")
|
||||
return val
|
||||
$$;
|
||||
|
||||
SELECT test1string('"a"'::jsonb);
|
||||
|
||||
-- test jsonb null -> python None
|
||||
CREATE FUNCTION test1null(val jsonb) RETURNS int
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
assert(val == None)
|
||||
return 1
|
||||
$$;
|
||||
|
||||
SELECT test1null('null'::jsonb);
|
||||
|
||||
-- test python -> jsonb
|
||||
CREATE FUNCTION roundtrip(val jsonb) RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
as $$
|
||||
return val
|
||||
$$;
|
||||
|
||||
SELECT roundtrip('null'::jsonb);
|
||||
SELECT roundtrip('1'::jsonb);
|
||||
SELECT roundtrip('1234567890.0987654321'::jsonb);
|
||||
SELECT roundtrip('-1234567890.0987654321'::jsonb);
|
||||
SELECT roundtrip('true'::jsonb);
|
||||
SELECT roundtrip('"string"'::jsonb);
|
||||
|
||||
SELECT roundtrip('{"1": null}'::jsonb);
|
||||
SELECT roundtrip('{"1": 1}'::jsonb);
|
||||
SELECT roundtrip('{"1": true}'::jsonb);
|
||||
SELECT roundtrip('{"1": "string"}'::jsonb);
|
||||
|
||||
SELECT roundtrip('[null]'::jsonb);
|
||||
SELECT roundtrip('[1]'::jsonb);
|
||||
SELECT roundtrip('[true]'::jsonb);
|
||||
SELECT roundtrip('["string"]'::jsonb);
|
||||
SELECT roundtrip('[null, 1]'::jsonb);
|
||||
SELECT roundtrip('[1, true]'::jsonb);
|
||||
SELECT roundtrip('[true, "string"]'::jsonb);
|
||||
SELECT roundtrip('["string", "string2"]'::jsonb);
|
||||
|
||||
-- test python infinity -> jsonb
|
||||
CREATE FUNCTION test1inf() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = float('inf')
|
||||
print(x)
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT test1inf();
|
||||
|
||||
-- test python -infinity -> jsonb
|
||||
CREATE FUNCTION test2inf() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = float('-inf')
|
||||
print(x)
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT test2inf();
|
||||
|
||||
-- test python NaN -> jsonb
|
||||
CREATE FUNCTION test1nan() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = float('nan')
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT test1nan();
|
||||
|
||||
-- complex numbers -> jsonb
|
||||
CREATE FUNCTION testComplexNumbers() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = 1 + 2j
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT testComplexNumbers();
|
||||
|
||||
-- range -> jsonb
|
||||
CREATE FUNCTION testRange() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = range(3)
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT testRange();
|
||||
|
||||
-- 0xff -> jsonb
|
||||
CREATE FUNCTION testDecimal() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = 0xff
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT testDecimal();
|
||||
|
||||
-- tuple -> jsonb
|
||||
CREATE FUNCTION testTuple() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = (1, 'String', None)
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT testTuple();
|
||||
|
||||
-- interesting dict -> jsonb
|
||||
CREATE FUNCTION test_dict1() RETURNS jsonb
|
||||
LANGUAGE plpythonu
|
||||
TRANSFORM FOR TYPE jsonb
|
||||
AS $$
|
||||
x = {"a": 1, None: 2, 33: 3}
|
||||
return x
|
||||
$$;
|
||||
|
||||
SELECT test_dict1();
|
||||
|
|
@ -569,4 +569,19 @@ SELECT jdoc->'guid', jdoc->'name' FROM api WHERE jdoc @> '{"tags": ["qu
|
|||
compared using the default database collation.
|
||||
</para>
|
||||
</sect2>
|
||||
|
||||
<sect2>
|
||||
<title>Transforms</title>
|
||||
|
||||
<para>
|
||||
Additional extensions are available that implement transforms for the
|
||||
<type>jsonb</type> type for the language PL/Python. The extensions for
|
||||
PL/Python are called <literal>jsonb_plpythonu</literal>,
|
||||
<literal>jsonb_plpython2u</literal>, and
|
||||
<literal>jsonb_plpython3u</literal> (see <xref
|
||||
linkend="plpython-python23"/> for the PL/Python naming convention). If you
|
||||
use them, <type>jsonb</type> values are mapped to Python dictionaries,
|
||||
lists, and scalars, as appropriate.
|
||||
</para>
|
||||
</sect2>
|
||||
</sect1>
|
||||
|
|
|
|||
Loading…
Reference in New Issue