hcq/openGauss-third_party
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6.0.0
openGauss-third_party/dependency/madlib/madlib.patch
yukai_k 056a6c6075 三方库构建源码
2020-12-31 15:28:51 +08:00

31019 lines
1.7 MiB
Raw Permalink Blame History

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5ea7732..2f34f87 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,6 +19,8 @@ cmake_minimum_required(VERSION 2.8.4 FATAL_ERROR)
include(ExternalProject)
include(CheckCXXCompilerFlag)
+ADD_DEFINITIONS(-DPGXC)
+ADD_DEFINITIONS(-DMADLIB)
# -- Local definitions (filenames, paths, etc.) --------------------------------
@@ -70,7 +72,7 @@ if(CMAKE_COMPILER_IS_GNUCC)
# http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_01
# We specify that we are POSIX.1-2001 compliant and XSI-conforming. We only
# need to specify _XOPEN_SOURCE as _POSIX_C_SOURCE will be set implicitly.
- set(CMAKE_C_FLAGS "-std=c99 -D_GLIBCXX_USE_CXX11_ABI=0 -pedantic -Wall -Wextra -Wno-clobbered -D_XOPEN_SOURCE=600"
+ set(CMAKE_C_FLAGS "-std=c99 -D_GLIBCXX_USE_CXX11_ABI=0 -fsigned-char -fpermissive -pedantic -Wall -Wextra -Wno-clobbered -D_XOPEN_SOURCE=600"
CACHE STRING
"Flags used by the compiler during all build types." FORCE)
if((CMAKE_C_COMPILER_VERSION VERSION_EQUAL 5.0) OR (CMAKE_C_COMPILER_VERSION VERSION_GREATER 5.0))
@@ -106,7 +108,7 @@ if(CMAKE_COMPILER_IS_GNUCXX)
# - long long is not part of the C++ 1998/2003 standard, but it is such a
# common (and useful) extension that we do not want to hear warnings about
# it.
- set(CMAKE_CXX_FLAGS "-std=gnu++98 -D_GLIBCXX_USE_CXX11_ABI=0 -fdiagnostics-show-option -Wall -Wextra -pedantic -Wconversion -Wno-long-long -Wno-clobbered ${SSE_DISABLE_OPTIONS} -fstrict-aliasing"
+ set(CMAKE_CXX_FLAGS "-std=gnu++98 -D_GLIBCXX_USE_CXX11_ABI=0 -fsigned-char -fpermissive -fdiagnostics-show-option -Wall -Wextra -pedantic -Wconversion -Wno-long-long -Wno-clobbered ${SSE_DISABLE_OPTIONS} -fstrict-aliasing"
CACHE STRING
"Flags used by the compiler during all build types." FORCE)
diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
index 4a6e496..707ec3b 100644
--- a/cmake/Utils.cmake
+++ b/cmake/Utils.cmake
@@ -22,7 +22,7 @@ function(define_m4_macros OUT_M4_CMD_LINE OUT_M4_CODE)
"__DBMS_VERSION_MAJOR__=${${DBMS_UC}_VERSION_MAJOR}"
"__DBMS_VERSION_MINOR__=${${DBMS_UC}_VERSION_MINOR}"
"__DBMS_VERSION_PATCH__=${${DBMS_UC}_VERSION_PATCH}"
- "__DBMS_ARCHITECTURE__=${${DBMS_UC}_ARCHITECTURE}"
+ "__DBMS_ARCHITECTURE__=x86_64"
"__MADLIB_VERSION__=${MADLIB_VERSION_STRING}"
"__MADLIB_VERSION_MAJOR__=${MADLIB_VERSION_MAJOR}"
"__MADLIB_VERSION_MINOR__=${MADLIB_VERSION_MINOR}"
diff --git a/methods/array_ops/src/pg_gp/array_ops.c b/methods/array_ops/src/pg_gp/array_ops.cpp
similarity index 97%
rename from methods/array_ops/src/pg_gp/array_ops.c
rename to methods/array_ops/src/pg_gp/array_ops.cpp
index a842a60..936fd39 100644
--- a/methods/array_ops/src/pg_gp/array_ops.c
+++ b/methods/array_ops/src/pg_gp/array_ops.cpp
@@ -20,6 +20,46 @@
PG_MODULE_MAGIC;
#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+Datum array_of_float(PG_FUNCTION_ARGS);
+Datum array_of_bigint(PG_FUNCTION_ARGS);
+Datum array_stddev(PG_FUNCTION_ARGS);
+Datum array_mean(PG_FUNCTION_ARGS);
+Datum array_sum_big(PG_FUNCTION_ARGS);
+Datum array_sum(PG_FUNCTION_ARGS);
+Datum array_abs_sum(PG_FUNCTION_ARGS);
+Datum array_min(PG_FUNCTION_ARGS);
+Datum array_max(PG_FUNCTION_ARGS);
+Datum array_max_index(PG_FUNCTION_ARGS);
+Datum array_min_index(PG_FUNCTION_ARGS);
+Datum array_dot(PG_FUNCTION_ARGS);
+Datum array_contains(PG_FUNCTION_ARGS);
+Datum array_add(PG_FUNCTION_ARGS);
+Datum array_sub(PG_FUNCTION_ARGS);
+Datum array_mult(PG_FUNCTION_ARGS);
+Datum array_div(PG_FUNCTION_ARGS);
+Datum array_abs(PG_FUNCTION_ARGS);
+Datum array_sqrt(PG_FUNCTION_ARGS);
+Datum array_pow(PG_FUNCTION_ARGS);
+Datum array_square(PG_FUNCTION_ARGS);
+Datum array_fill_(PG_FUNCTION_ARGS);
+Datum array_cos(PG_FUNCTION_ARGS);
+Datum array_scalar_mult(PG_FUNCTION_ARGS);
+Datum array_scalar_add(PG_FUNCTION_ARGS);
+Datum array_cum_sum(PG_FUNCTION_ARGS);
+Datum array_cum_prod(PG_FUNCTION_ARGS);
+Datum array_filter(PG_FUNCTION_ARGS);
+Datum array_normalize(PG_FUNCTION_ARGS);
+Datum array_contains_null(PG_FUNCTION_ARGS);
+Datum array_to_bytea(PG_FUNCTION_ARGS);
+
+#ifdef __cplusplus
+}
+#endif
/*
This module provides c implementations for several postgres array operations.
@@ -378,7 +418,7 @@ array_of_float(PG_FUNCTION_ARGS){
errmsg("invalid array length"),
errdetail("array_of_float: Size should be in [1, 1e7], %d given", size)));
}
- Datum* array = palloc (sizeof(Datum)*size);
+ Datum* array = (Datum *) palloc (sizeof(Datum)*size);
for(int i = 0; i < size; ++i) {
array[i] = Float8GetDatum(0);
}
@@ -405,7 +445,7 @@ array_of_bigint(PG_FUNCTION_ARGS){
errmsg("invalid array length"),
errdetail("array_of_bigint: Size should be in [1, 1e7], %d given", size)));
}
- Datum* array = palloc (sizeof(Datum)*size);
+ Datum* array = (Datum *) palloc (sizeof(Datum)*size);
for(int i = 0; i < size; ++i) {
array[i] = Int64GetDatum(0);
}
@@ -823,9 +863,9 @@ array_square(PG_FUNCTION_ARGS){
/*
* This function sets all elements to be the specified value.
*/
-PG_FUNCTION_INFO_V1(array_fill);
+PG_FUNCTION_INFO_V1(array_fill_);
Datum
-array_fill(PG_FUNCTION_ARGS){
+array_fill_(PG_FUNCTION_ARGS){
if (PG_ARGISNULL(0)) { PG_RETURN_NULL(); }
if (PG_ARGISNULL(1)) { PG_RETURN_NULL(); }
@@ -2129,7 +2169,7 @@ array_to_bytea(PG_FUNCTION_ARGS)
data_length = (nitems * TI->typlen);
}
- bytea *ba = palloc(VARHDRSZ + data_length);
+ bytea *ba = (bytea *)palloc(VARHDRSZ + data_length);
SET_VARSIZE(ba, VARHDRSZ + data_length);
diff --git a/methods/array_ops/src/pg_gp/array_ops.sql_in b/methods/array_ops/src/pg_gp/array_ops.sql_in
index c1ec853..fc7036b 100644
--- a/methods/array_ops/src/pg_gp/array_ops.sql_in
+++ b/methods/array_ops/src/pg_gp/array_ops.sql_in
@@ -268,7 +268,7 @@ File array_ops.sql_in for list of functions and usage.
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_add(x anyarray, y anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_add'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -295,7 +295,7 @@ CREATE AGGREGATE MADLIB_SCHEMA.sum(/* x */ anyarray) (
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sub(x anyarray, y anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_sub'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -308,7 +308,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_mult(x anyarray, y anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_mult'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -321,7 +321,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_div(x anyarray, y anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_div'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -334,7 +334,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_dot(x anyarray, y anyarray) RETURNS float8
AS 'MODULE_PATHNAME', 'array_dot'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -347,7 +347,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_contains(x anyarray, y anyarray) RETURNS BOOL
AS 'MODULE_PATHNAME', 'array_contains'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -359,7 +359,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_max(x anyarray) RETURNS anyelement
AS 'MODULE_PATHNAME', 'array_max'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -371,7 +371,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_max_index(x anyarray) RETURNS float8[]
AS 'MODULE_PATHNAME', 'array_max_index'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -383,7 +383,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_min(x anyarray) RETURNS anyelement
AS 'MODULE_PATHNAME', 'array_min'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -395,7 +395,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_min_index(x anyarray) RETURNS float8[]
AS 'MODULE_PATHNAME', 'array_min_index'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -407,7 +407,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sum(x anyarray) RETURNS anyelement
AS 'MODULE_PATHNAME', 'array_sum'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -420,7 +420,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sum_big(x anyarray) RETURNS float8
AS 'MODULE_PATHNAME', 'array_sum_big'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -431,7 +431,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_abs_sum(x anyarray) RETURNS anyelement
AS 'MODULE_PATHNAME', 'array_abs_sum'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -443,7 +443,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_abs(x anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_abs'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -455,7 +455,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_mean(x anyarray) RETURNS float8
AS 'MODULE_PATHNAME', 'array_mean'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -467,7 +467,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_stddev(x anyarray) RETURNS float8
AS 'MODULE_PATHNAME', 'array_stddev'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -479,7 +479,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_of_float(k integer) RETURNS float8[]
AS 'MODULE_PATHNAME', 'array_of_float'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -491,7 +491,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_of_bigint(k integer) RETURNS bigint[]
AS 'MODULE_PATHNAME', 'array_of_bigint'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -503,8 +503,8 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_fill(x anyarray, k anyelement) RETURNS anyarray
-AS 'MODULE_PATHNAME', 'array_fill'
-LANGUAGE C IMMUTABLE
+AS 'MODULE_PATHNAME', 'array_fill_'
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -515,7 +515,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_cos(x anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_cos'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -527,7 +527,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_scalar_mult(x anyarray, k anyelement) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_scalar_mult'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -539,7 +539,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_scalar_add(x anyarray, k anyelement) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_scalar_add'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -551,7 +551,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
* @returns Filtered array.
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_filter(x anyarray, k anyelement, op text) RETURNS anyarray
-AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE
+AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -561,7 +561,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
* @returns Filtered array.
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_filter(x anyarray, k anyelement) RETURNS anyarray
-AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE
+AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -570,7 +570,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
* @returns Filtered array.
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_filter(x anyarray) RETURNS anyarray
-AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE
+AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -583,7 +583,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_sqrt(x anyarray) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.array_sqrt(x anyarray) RETURNS float8[]
AS 'MODULE_PATHNAME', 'array_sqrt'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -597,7 +597,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_pow(x anyarray, y anyelement) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.array_pow(x anyarray, y anyelement) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_pow'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -610,7 +610,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_square(x anyarray) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.array_square(x anyarray) RETURNS float8[]
AS 'MODULE_PATHNAME', 'array_square'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -622,7 +622,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.normalize(float8[]) CASCADE;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalize(x anyarray) RETURNS float8[]
-AS 'MODULE_PATHNAME', 'array_normalize' LANGUAGE C IMMUTABLE STRICT
+AS 'MODULE_PATHNAME', 'array_normalize' LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -634,7 +634,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_contains_null(float8[]) CASCADE;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_contains_null(x anyarray) RETURNS BOOLEAN
-AS 'MODULE_PATHNAME', 'array_contains_null' LANGUAGE C IMMUTABLE STRICT
+AS 'MODULE_PATHNAME', 'array_contains_null' LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -648,7 +648,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_cum_sum(x anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_cum_sum'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -662,7 +662,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_cum_prod(x anyarray) RETURNS anyarray
AS 'MODULE_PATHNAME', 'array_cum_prod'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -741,7 +741,115 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_to_bytea(ANYARRAY)
RETURNS BYTEA
AS
'MODULE_PATHNAME', 'array_to_bytea'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED;
!>)
m4_changequote(`,')
m4_ifdef(__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sort(x ANYARRAY) RETURNS ANYARRAY AS
+$BODY$
+SELECT ARRAY(
+ SELECT $1[s.i] AS "foo"
+ FROM
+ generate_series(array_lower($1,1), array_upper($1,1)) AS s(i)
+ ORDER BY foo);
+$BODY$ LANGUAGE SQL IMMUTABLE;
+------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.percentile_cont(myarray anyarray, percentile double precision)RETURNS double precision AS $$
+DECLARE
+ ary_cnt bigint;
+ row_num bigint;
+ crn double precision;
+ frn double precision;
+ calc_result double precision;
+ new_array double precision[];
+BEGIN
+ new_array := array_sort(myarray);
+ ary_cnt := array_length(new_array,1);
+ row_num := 1 + ( percentile * ( ary_cnt - 1 ));
+ crn := ceiling(row_num);
+ frn := floor(row_num);
+ if crn = frn and frn = row_num then
+ calc_result := new_array[row_num];
+ else
+ calc_result := (crn - row_num) * new_array[frn] + (row_num - frn) * new_array[crn];
+ end if;
+ RETURN calc_result;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.percentile_disc(myarray anyarray, percentile double precision) RETURNS double precision AS $$
+DECLARE
+ ary_cnt bigint;
+ row_num double precision;
+ frn bigint;
+ calc_result double precision;
+ new_array double precision[];
+BEGIN
+ new_array := array_sort(myarray);
+ ary_cnt := array_length(new_array,1);
+ row_num := percentile * ary_cnt;
+ frn := ceil(row_num);
+ calc_result := new_array[frn];
+ RETURN calc_result;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-----------------------------------------------------------------
+-- aggregation for independent var
+DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(REAL[]);
+CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(REAL[]) (
+ SFUNC = array_cat,
+ PREFUNC = array_cat,
+ STYPE = REAL[]
+
+ );
+
+-- aggregation for dependent var
+DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(SMALLINT[]);
+CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(SMALLINT[]) (
+ SFUNC = array_cat,
+ PREFUNC = array_cat,
+ STYPE = SMALLINT[]
+
+ );
+
+CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var REAL[])
+RETURNS BYTEA
+AS
+$$
+import numpy as np
+
+return np.array(var, dtype=np.float32).tobytes()
+$$ LANGUAGE plpythonu;
+
+CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var SMALLINT[])
+RETURNS BYTEA
+AS
+$$
+import numpy as np
+
+return np.array(var, dtype=np.int16).tobytes()
+$$ LANGUAGE plpythonu;
+
+
+CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_real_array(var BYTEA)
+RETURNS REAL[]
+AS
+$$
+import numpy as np
+
+return np.frombuffer(var, dtype=np.float32)
+$$ LANGUAGE plpythonu;
+
+
+CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_smallint_array(var BYTEA)
+RETURNS SMALLINT[]
+AS
+$$
+import numpy as np
+
+return np.frombuffer(var, dtype=np.int16)
+$$ LANGUAGE plpythonu;
diff --git a/methods/kmeans/src/pg_gp/kmeans.c b/methods/kmeans/src/pg_gp/kmeans.cpp
similarity index 95%
rename from methods/kmeans/src/pg_gp/kmeans.c
rename to methods/kmeans/src/pg_gp/kmeans.cpp
index d74f65b..a4785b0 100644
--- a/methods/kmeans/src/pg_gp/kmeans.c
+++ b/methods/kmeans/src/pg_gp/kmeans.cpp
@@ -17,6 +17,19 @@ typedef enum {
TANIMOTO
} KMeansMetric;
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum internal_get_array_of_close_canopies(PG_FUNCTION_ARGS);
+Datum internal_kmeans_closest_centroid(PG_FUNCTION_ARGS);
+Datum internal_kmeans_agg_centroid_trans(PG_FUNCTION_ARGS);
+Datum internal_kmeans_agg_centroid_merge(PG_FUNCTION_ARGS);
+Datum internal_kmeans_canopy_transition(PG_FUNCTION_ARGS);
+Datum internal_remove_close_canopies(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
static
inline
int
@@ -128,7 +141,7 @@ internal_get_array_of_close_canopies(PG_FUNCTION_ARGS)
get_svec_array_elms(PG_GETARG_ARRAYTYPE_P(verify_arg_nonnull(fcinfo, 1)),
&all_canopies, &num_all_canopies);
threshold = PG_GETARG_FLOAT8(verify_arg_nonnull(fcinfo, 2));
- metric_fn = get_metric_fn(PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 3)));
+ metric_fn = get_metric_fn((KMeansMetric)PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 3)));
mem_context_for_function_calls = setup_mem_context_for_functional_calls();
close_canopies = (int32 *) palloc(sizeof(int32) * num_all_canopies);
@@ -382,7 +395,7 @@ internal_kmeans_closest_centroid(PG_FUNCTION_ARGS) {
cid = indirect ? canopy_ids[i] - ARR_LBOUND(canopy_ids_arr)[0] : i;
double * centroid = c_centroids_array+cid*dimension;
- MetricFunc func = get_metric_fn_for_array(dist_metric);
+ MetricFunc func = get_metric_fn_for_array((KMeansMetric)dist_metric);
distance = (*func)(centroid, c_point_array, dimension);
if (distance < min_distance) {
@@ -459,7 +472,7 @@ internal_kmeans_agg_centroid_trans(PG_FUNCTION_ARGS) {
if (PG_ARGISNULL(0))
{
- c_array = palloc0(expected_array_len*sizeof(float8));
+ c_array = (float8 *) palloc0(expected_array_len*sizeof(float8));
rebuild_array = true;
}
else
@@ -565,7 +578,7 @@ internal_kmeans_canopy_transition(PG_FUNCTION_ARGS) {
canopies_arr = PG_GETARG_ARRAYTYPE_P(verify_arg_nonnull(fcinfo, 0));
get_svec_array_elms(canopies_arr, &canopies, &num_canopies);
point = PG_GETARG_SVECTYPE_P(verify_arg_nonnull(fcinfo, 1));
- metric_fn = get_metric_fn(PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 2)));
+ metric_fn = get_metric_fn((KMeansMetric)PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 2)));
threshold = PG_GETARG_FLOAT8(verify_arg_nonnull(fcinfo, 3));
mem_context_for_function_calls = setup_mem_context_for_functional_calls();
@@ -609,7 +622,7 @@ internal_remove_close_canopies(PG_FUNCTION_ARGS) {
all_canopies_arr = PG_GETARG_ARRAYTYPE_P(verify_arg_nonnull(fcinfo, 0));
get_svec_array_elms(all_canopies_arr, &all_canopies, &num_all_canopies);
- metric_fn = get_metric_fn(PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 1)));
+ metric_fn = get_metric_fn((KMeansMetric)PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 1)));
threshold = PG_GETARG_FLOAT8(verify_arg_nonnull(fcinfo, 2));
mem_context_for_function_calls = setup_mem_context_for_functional_calls();
diff --git a/methods/sketch/src/pg_gp/countmin.c b/methods/sketch/src/pg_gp/countmin.cpp
similarity index 97%
rename from methods/sketch/src/pg_gp/countmin.c
rename to methods/sketch/src/pg_gp/countmin.cpp
index 8b70ec6..152d51a 100644
--- a/methods/sketch/src/pg_gp/countmin.c
+++ b/methods/sketch/src/pg_gp/countmin.cpp
@@ -39,6 +39,17 @@
#include "sketch_support.h"
#include "countmin.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum __cmsketch_int8_trans(PG_FUNCTION_ARGS);
+Datum __cmsketch_base64_final(PG_FUNCTION_ARGS);
+Datum __cmsketch_merge(PG_FUNCTION_ARGS);
+Datum cmsketch_dump(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
PG_FUNCTION_INFO_V1(__cmsketch_int8_trans);
/*
@@ -248,7 +259,7 @@ Datum __cmsketch_base64_final(PG_FUNCTION_ARGS)
elog(ERROR, "invalid transition state for cmsketch");
}
- out = palloc0(len);
+ out = (bytea *) palloc0(len);
if (VARSIZE(blob) > VARHDRSZ) {
sketch = (cmtransval *)VARDATA(blob);
memcpy((uint8 *)VARDATA(out), sketch->sketches, len - VARHDRSZ);
@@ -263,7 +274,7 @@ Datum __cmsketch_base64_final(PG_FUNCTION_ARGS)
res;
datalen = VARSIZE(data) - VARHDRSZ;
resultlen = b64_enc_len(datalen);
- result = palloc(VARHDRSZ + resultlen);
+ result = (text *) palloc(VARHDRSZ + resultlen);
res = b64_encode(VARDATA(data), datalen, VARDATA(result));
/* Make this FATAL 'cause we've trodden on memory ... */
if (res > resultlen)
diff --git a/methods/sketch/src/pg_gp/countmin.h b/methods/sketch/src/pg_gp/countmin.h
index 28f1347..11476d8 100644
--- a/methods/sketch/src/pg_gp/countmin.h
+++ b/methods/sketch/src/pg_gp/countmin.h
@@ -172,6 +172,10 @@ int cnt_cmp_desc(const void *i, const void *j);
/* UDF protos */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
Datum __cmsketch_int8_trans(PG_FUNCTION_ARGS);
Datum cmsketch_width_histogram(PG_FUNCTION_ARGS);
Datum cmsketch_dhistogram(PG_FUNCTION_ARGS);
@@ -187,5 +191,8 @@ Datum __mfvsketch_trans(PG_FUNCTION_ARGS);
Datum __mfvsketch_final(PG_FUNCTION_ARGS);
Datum __mfvsketch_merge(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
#endif /* _COUNTMIN_H_ */
diff --git a/methods/sketch/src/pg_gp/fm.c b/methods/sketch/src/pg_gp/fm.cpp
similarity index 98%
rename from methods/sketch/src/pg_gp/fm.c
rename to methods/sketch/src/pg_gp/fm.cpp
index 39f3413..c8b8011 100644
--- a/methods/sketch/src/pg_gp/fm.c
+++ b/methods/sketch/src/pg_gp/fm.cpp
@@ -104,6 +104,17 @@ typedef struct {
char storage[];
} fmtransval;
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum __fmsketch_trans(PG_FUNCTION_ARGS);
+Datum __fmsketch_count_distinct(PG_FUNCTION_ARGS);
+Datum __fmsketch_merge(PG_FUNCTION_ARGS);
+Datum big_or(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
/* check whter the contents in fmtransval::storage is safe for sortasort */
void check_sortasort(sortasort *st, size_t st_size) {
size_t left_len = st_size;
@@ -317,7 +328,7 @@ Datum __fmsketch_trans(PG_FUNCTION_ARGS)
* zero bits for an empty FM sketch.
* \param template an optional pre-existing transval whose fields we can copy in
*/
-bytea *fm_new(fmtransval *template)
+bytea *fm_new(fmtransval *template1)
{
int fmsize = VARHDRSZ + sizeof(fmtransval) + FMSKETCH_SZ;
/* use palloc0 to make sure it's initialized to 0 */
@@ -327,8 +338,8 @@ bytea *fm_new(fmtransval *template)
SET_VARSIZE(newblob, fmsize);
transval = (fmtransval *)VARDATA(newblob);
/* copy over the struct values */
- if (template != NULL)
- memcpy(transval, template, sizeof(fmtransval));
+ if (template1 != NULL)
+ memcpy(transval, template1, sizeof(fmtransval));
/* set status to BIG, possibly overwriting what was in template */
transval->status = BIG;
diff --git a/methods/sketch/src/pg_gp/mfvsketch.c b/methods/sketch/src/pg_gp/mfvsketch.cpp
similarity index 98%
rename from methods/sketch/src/pg_gp/mfvsketch.c
rename to methods/sketch/src/pg_gp/mfvsketch.cpp
index 57e8c5d..6192ad6 100644
--- a/methods/sketch/src/pg_gp/mfvsketch.c
+++ b/methods/sketch/src/pg_gp/mfvsketch.cpp
@@ -45,6 +45,16 @@
#include <ctype.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum __mfvsketch_trans(PG_FUNCTION_ARGS);
+Datum __mfvsketch_final(PG_FUNCTION_ARGS);
+Datum __mfvsketch_merge(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
/* check whether the content in the given bytea is safe for mfvtransval */
void check_mfvtransval(bytea *storage) {
size_t left_len = VARSIZE(storage);
@@ -515,7 +525,7 @@ bytea *mfvsketch_merge_c(bytea *transblob1, bytea *transblob2)
{
mfvtransval *transval1 = (mfvtransval *)VARDATA(transblob1);
mfvtransval *transval2 = (mfvtransval *)VARDATA(transblob2);
- void *newblob;
+ bytea *newblob;
mfvtransval *newval;
uint32 i, j, cnt;
@@ -572,7 +582,7 @@ bytea *mfvsketch_merge_c(bytea *transblob1, bytea *transblob2)
qsort(transval1->mfvs, transval1->next_mfv, sizeof(offsetcnt), cnt_cmp_desc);
qsort(transval2->mfvs, transval2->next_mfv, sizeof(offsetcnt), cnt_cmp_desc);
- Datum *values_seen = palloc0((2 * newval->max_mfvs) * sizeof(Datum));
+ Datum *values_seen = (Datum *)palloc0((2 * newval->max_mfvs) * sizeof(Datum));
unsigned int k;
/* choose top k from transval1 and transval2 */
diff --git a/methods/sketch/src/pg_gp/sketch.sql_in b/methods/sketch/src/pg_gp/sketch.sql_in
index 2ae36a2..60e4ae8 100644
--- a/methods/sketch/src/pg_gp/sketch.sql_in
+++ b/methods/sketch/src/pg_gp/sketch.sql_in
@@ -381,28 +381,28 @@ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.big_or(bitmap1 bytea, bitmap2 bytea) CASCA
CREATE FUNCTION MADLIB_SCHEMA.big_or(bitmap1 bytea, bitmap2 bytea)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_trans(bitmaps bytea, input anyelement) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_trans(bitmaps bytea, input anyelement)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_count_distinct(bitmaps bytea) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_count_distinct(bitmaps bytea)
RETURNS int8
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_merge(bitmaps1 bytea, bitmaps2 bytea) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_merge(bitmaps1 bytea, bitmaps2 bytea)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fmsketch_dcount(anyelement);
@@ -431,42 +431,42 @@ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8) CASCADE
CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8, int8) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8, arg2 int8)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8, int8, int8) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8, arg2 int8, arg3 int8)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_base64_final(bytea) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_base64_final(sketch bytea)
RETURNS text
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_merge(bytea, bytea) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_merge(bytea, bytea)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.cmsketch(int8);
@@ -580,33 +580,33 @@ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_trans(bytea, anyelement, int4)
CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_trans(bytea, anyelement, int4)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_final(bytea) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_final(bytea)
RETURNS text[][]
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_merge(bytea, bytea) CASCADE;
CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_merge(bytea, bytea)
RETURNS bytea
AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sketch_rightmost_one(bytea, integer, integer)
-RETURNS integer AS 'MODULE_PATHNAME', 'sketch_rightmost_one' LANGUAGE C STRICT
+RETURNS integer AS 'MODULE_PATHNAME', 'sketch_rightmost_one' LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sketch_leftmost_zero(bytea, integer, integer)
-RETURNS integer AS 'MODULE_PATHNAME', 'sketch_leftmost_zero' LANGUAGE C STRICT
+RETURNS integer AS 'MODULE_PATHNAME', 'sketch_leftmost_zero' LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sketch_array_set_bit_in_place(bytea, integer, integer, integer, integer)
-RETURNS bytea AS 'MODULE_PATHNAME', 'sketch_array_set_bit_in_place' LANGUAGE C STRICT
+RETURNS bytea AS 'MODULE_PATHNAME', 'sketch_array_set_bit_in_place' LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.mfvsketch_top_histogram( anyelement, int4);
diff --git a/methods/sketch/src/pg_gp/sketch_support.c b/methods/sketch/src/pg_gp/sketch_support.cpp
similarity index 98%
rename from methods/sketch/src/pg_gp/sketch_support.c
rename to methods/sketch/src/pg_gp/sketch_support.cpp
index 1cdeb90..ca57b07 100644
--- a/methods/sketch/src/pg_gp/sketch_support.c
+++ b/methods/sketch/src/pg_gp/sketch_support.cpp
@@ -50,7 +50,15 @@
#include <utils/lsyscache.h>
#include "sketch_support.h"
-
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum sketch_array_set_bit_in_place(PG_FUNCTION_ARGS);
+Datum sketch_rightmost_one(PG_FUNCTION_ARGS);
+Datum sketch_leftmost_zero(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
/*!
* Simple linear function to find the rightmost bit that's set to one
@@ -302,7 +310,7 @@ bytea *sketch_md5_bytea(Datum dat, Oid typOid)
// according to postgres' libpq/md5.c, need 33 bytes to hold
// null-terminated md5 string
char outbuf[MD5_HASHLEN*2+1];
- bytea *out = palloc0(MD5_HASHLEN+VARHDRSZ);
+ bytea *out = (bytea *) palloc0(MD5_HASHLEN+VARHDRSZ);
bool byval = get_typbyval(typOid);
int len = ExtractDatumLen(dat, get_typlen(typOid), byval, -1);
void *datp = DatumExtractPointer(dat, byval);
diff --git a/methods/sketch/src/pg_gp/sortasort.c b/methods/sketch/src/pg_gp/sortasort.cpp
similarity index 99%
rename from methods/sketch/src/pg_gp/sortasort.c
rename to methods/sketch/src/pg_gp/sortasort.cpp
index c052ca7..1bc9a72 100644
--- a/methods/sketch/src/pg_gp/sortasort.c
+++ b/methods/sketch/src/pg_gp/sortasort.cpp
@@ -30,6 +30,13 @@
#define SORTASORT_DATA(s) (((char *)(s->dir)) + \
(s->capacity * (sizeof (s->dir[0]))))
+#ifdef __cplusplus
+extern "C" {
+#endif
+#ifdef __cplusplus
+}
+#endif
+
/*
* get the ith item stored in sortasort
*/
diff --git a/methods/stemmer/src/pg_gp/porter_stemmer.c b/methods/stemmer/src/pg_gp/porter_stemmer.cpp
similarity index 97%
rename from methods/stemmer/src/pg_gp/porter_stemmer.c
rename to methods/stemmer/src/pg_gp/porter_stemmer.cpp
index 598e232..2a65862 100644
--- a/methods/stemmer/src/pg_gp/porter_stemmer.c
+++ b/methods/stemmer/src/pg_gp/porter_stemmer.cpp
@@ -61,6 +61,15 @@
*/
#include "libstemmer/include/libstemmer.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum stem_token(PG_FUNCTION_ARGS);
+Datum stem_token_arr(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
static const char * stem_token_text(struct sb_stemmer *stemmer, text *token) {
char * stemmed_token = TextDatumGetCString(PointerGetDatum(token));
if (strcmp(stemmed_token, "") == 0) {
diff --git a/methods/stemmer/src/pg_gp/porter_stemmer.sql_in b/methods/stemmer/src/pg_gp/porter_stemmer.sql_in
index 43e2805..1e39b5f 100644
--- a/methods/stemmer/src/pg_gp/porter_stemmer.sql_in
+++ b/methods/stemmer/src/pg_gp/porter_stemmer.sql_in
@@ -172,7 +172,7 @@ File porter_stemmer.sql_in for list of functions and usage.
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token(token text) RETURNS text
AS 'MODULE_PATHNAME', 'stem_token'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -184,7 +184,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token_arr(token_arr text[]) RETURNS text[]
AS 'MODULE_PATHNAME', 'stem_token_arr'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/methods/svec/src/pg_gp/SparseData.c b/methods/svec/src/pg_gp/SparseData.cpp
similarity index 100%
rename from methods/svec/src/pg_gp/SparseData.c
rename to methods/svec/src/pg_gp/SparseData.cpp
diff --git a/methods/svec/src/pg_gp/operators.c b/methods/svec/src/pg_gp/operators.cpp
similarity index 93%
rename from methods/svec/src/pg_gp/operators.c
rename to methods/svec/src/pg_gp/operators.cpp
index fb83998..4c41fd5 100644
--- a/methods/svec/src/pg_gp/operators.c
+++ b/methods/svec/src/pg_gp/operators.cpp
@@ -24,6 +24,65 @@
PG_MODULE_MAGIC;
#endif
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum svec_concat_replicate(PG_FUNCTION_ARGS);
+Datum svec_concat(PG_FUNCTION_ARGS);
+Datum svec_cmp(PG_FUNCTION_ARGS);
+Datum svec_ge(PG_FUNCTION_ARGS);
+Datum svec_lt(PG_FUNCTION_ARGS);
+Datum svec_le(PG_FUNCTION_ARGS);
+Datum svec_gt(PG_FUNCTION_ARGS);
+Datum svec_ne(PG_FUNCTION_ARGS);
+Datum svec_eq(PG_FUNCTION_ARGS);
+Datum svec_l2_cmp(PG_FUNCTION_ARGS);
+Datum svec_l2_lt(PG_FUNCTION_ARGS);
+Datum svec_l2_le(PG_FUNCTION_ARGS);
+Datum svec_l2_eq(PG_FUNCTION_ARGS);
+Datum svec_l2_ne(PG_FUNCTION_ARGS);
+Datum svec_l2_gt(PG_FUNCTION_ARGS);
+Datum svec_l2_ge(PG_FUNCTION_ARGS);
+Datum svec_pow(PG_FUNCTION_ARGS);
+Datum svec_minus(PG_FUNCTION_ARGS);
+Datum svec_plus(PG_FUNCTION_ARGS);
+Datum svec_mult(PG_FUNCTION_ARGS);
+Datum svec_div(PG_FUNCTION_ARGS);
+Datum svec_dot(PG_FUNCTION_ARGS);
+Datum svec_cast_int2(PG_FUNCTION_ARGS);
+Datum svec_cast_int4(PG_FUNCTION_ARGS);
+Datum svec_cast_int8(PG_FUNCTION_ARGS);
+Datum svec_cast_float4(PG_FUNCTION_ARGS);
+Datum svec_cast_float8(PG_FUNCTION_ARGS);
+Datum svec_cast_numeric(PG_FUNCTION_ARGS);
+Datum float8arr_cast_int2(PG_FUNCTION_ARGS);
+Datum float8arr_cast_int4(PG_FUNCTION_ARGS);
+Datum float8arr_cast_int8(PG_FUNCTION_ARGS);
+Datum float8arr_cast_float4(PG_FUNCTION_ARGS);
+Datum float8arr_cast_float8(PG_FUNCTION_ARGS);
+Datum float8arr_cast_numeric(PG_FUNCTION_ARGS);
+Datum svec_cast_float8arr(PG_FUNCTION_ARGS);
+Datum svec_cast_positions_float8arr(PG_FUNCTION_ARGS);
+Datum float8arr_equals(PG_FUNCTION_ARGS);
+Datum float8arr_dot(PG_FUNCTION_ARGS);
+Datum float8arr_minus_float8arr(PG_FUNCTION_ARGS);
+Datum svec_minus_float8arr(PG_FUNCTION_ARGS);
+Datum float8arr_minus_svec(PG_FUNCTION_ARGS);
+Datum float8arr_plus_float8arr(PG_FUNCTION_ARGS);
+Datum svec_plus_float8arr(PG_FUNCTION_ARGS);
+Datum float8arr_plus_svec(PG_FUNCTION_ARGS);
+Datum float8arr_mult_float8arr(PG_FUNCTION_ARGS);
+Datum svec_mult_float8arr(PG_FUNCTION_ARGS);
+Datum float8arr_mult_svec(PG_FUNCTION_ARGS);
+Datum float8arr_div_float8arr(PG_FUNCTION_ARGS);
+Datum svec_div_float8arr(PG_FUNCTION_ARGS);
+Datum float8arr_div_svec(PG_FUNCTION_ARGS);
+Datum svec_dot_float8arr(PG_FUNCTION_ARGS);
+Datum float8arr_dot_svec(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
/**
* For many functions defined in this module, the operation has no meaning
* if the array dimensions aren't the same, unless one of the inputs is a
diff --git a/methods/svec/src/pg_gp/sparse_vector.c b/methods/svec/src/pg_gp/sparse_vector.cpp
similarity index 97%
rename from methods/svec/src/pg_gp/sparse_vector.c
rename to methods/svec/src/pg_gp/sparse_vector.cpp
index 7944419..f0177d6 100644
--- a/methods/svec/src/pg_gp/sparse_vector.c
+++ b/methods/svec/src/pg_gp/sparse_vector.cpp
@@ -27,6 +27,17 @@
#include "utils/memutils.h"
#include "sparse_vector.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum svec_send(PG_FUNCTION_ARGS);
+Datum svec_recv(PG_FUNCTION_ARGS);
+Datum svec_return_array(PG_FUNCTION_ARGS);
+Datum svec_out(PG_FUNCTION_ARGS);
+Datum svec_in(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
/**
* @return An array of float8s obtained by converting a given sparse vector
*/
diff --git a/methods/svec/src/pg_gp/sparse_vector.h b/methods/svec/src/pg_gp/sparse_vector.h
index e29185b..18a5171 100644
--- a/methods/svec/src/pg_gp/sparse_vector.h
+++ b/methods/svec/src/pg_gp/sparse_vector.h
@@ -104,6 +104,10 @@ void check_dimension(SvecType *svec1, SvecType *svec2, char *msg);
double svec_svec_dot_product(SvecType *svec1, SvecType *svec2);
SparseData sdata_uncompressed_from_float8arr_internal(ArrayType *array);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
Datum svec_in(PG_FUNCTION_ARGS);
Datum svec_out(PG_FUNCTION_ARGS);
Datum svec_return_vector(PG_FUNCTION_ARGS);
@@ -154,4 +158,8 @@ Datum float8arr_cast_numeric(PG_FUNCTION_ARGS);
Datum svec_cast_float8arr(PG_FUNCTION_ARGS);
Datum svec_cast_positions_float8arr(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
#endif /* SPARSEVECTOR_H */
diff --git a/methods/svec/src/pg_gp/svec.sql_in b/methods/svec/src/pg_gp/svec.sql_in
index 2d0e21e..4f484c0 100644
--- a/methods/svec/src/pg_gp/svec.sql_in
+++ b/methods/svec/src/pg_gp/svec.sql_in
@@ -634,7 +634,7 @@ CREATE TYPE MADLIB_SCHEMA.svec;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_in(cstring)
RETURNS MADLIB_SCHEMA.svec
AS 'MODULE_PATHNAME'
- LANGUAGE C IMMUTABLE STRICT
+ LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Converts SVEC to CSTRING.
@@ -642,7 +642,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_in(cstring)
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_out(MADLIB_SCHEMA.svec)
RETURNS cstring
AS 'MODULE_PATHNAME'
- LANGUAGE C IMMUTABLE STRICT
+ LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Converts SVEC internal representation to SVEC.
@@ -650,7 +650,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_out(MADLIB_SCHEMA.svec)
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_recv(internal)
RETURNS MADLIB_SCHEMA.svec
AS 'MODULE_PATHNAME'
- LANGUAGE C IMMUTABLE STRICT
+ LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Converts SVEC to BYTEA.
@@ -658,7 +658,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_recv(internal)
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_send(MADLIB_SCHEMA.svec)
RETURNS bytea
AS 'MODULE_PATHNAME'
- LANGUAGE C IMMUTABLE STRICT
+ LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE TYPE MADLIB_SCHEMA.svec (
@@ -673,232 +673,232 @@ CREATE TYPE MADLIB_SCHEMA.svec (
--! Adds two SVECs together, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Minus second SVEC from the first, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Divides the first SVEC by the second, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Multiplies two SVEVs together, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Raises each element of the first SVEC to the power given by second SVEC, which must have dimension 1 (a scalar).
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_pow(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_pow' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_pow(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_pow' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Compare two SVECs, -1 for left side smaller, 1 for greater, 0 for equal
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_cmp' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_cmp' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the left SVEC is greater or equal to the right.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ge' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ge' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the left SVEC is less than the right.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_lt' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_lt' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the left SVEC is less or equal to the right.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_le' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_le' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the left SVEC is greater.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_gt' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_gt' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if two SVECs are not equal. If the two SVECs are of different size, then will return true.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ne' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ne' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if two SVECs are equal. If the two SVECs are of different size, then will return false.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_eq' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_eq' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if two float8 arrays are equal
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_eq(float8[],float8[]) RETURNS boolean AS 'MODULE_PATHNAME', 'float8arr_equals' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_eq(float8[],float8[]) RETURNS boolean AS 'MODULE_PATHNAME', 'float8arr_equals' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Minus second array from the first array, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Minus second SVEC from the first array, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_svec' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_svec' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Minus second array from the first SVEC, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Adds two arrays together, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Adds an array and an SVEC, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_svec' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_svec' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Adds an SVEC and an array, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Multiplies two float8 arrays, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Multiplies an array and an SVEC, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_svec' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_svec' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Multiplies an SVEC and an array, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Divides a float8 array by another, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Divides a float8 array by an SVEC, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_svec' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_svec' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Divides an SVEC by a float8 array, element by element.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div_float8arr' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div_float8arr' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the dot product of two SVECs.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the dot product of two float8 arrays.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the dot product of an SVEC and a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot_float8arr' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot_float8arr' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the dot product of a float8 array and an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot_svec' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot_svec' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an int2 into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int2(int2) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int2' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int2(int2) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int2' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an int4 into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int4(int4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int4' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int4(int4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int4' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an int8 into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int8(bigint) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int8' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int8(bigint) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int8' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts a float4 into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float4(float4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float4' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float4(float4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float4' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts a float8 into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8(float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8(float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts a numeric into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_numeric(numeric) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_numeric' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_numeric(numeric) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_numeric' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an int2 into a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int2(int2) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int2' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int2(int2) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int2' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an int4 into a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int4(int4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int4' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int4(int4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int4' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an int8 into a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int8(bigint) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int8' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int8(bigint) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int8' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts a float4 into a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float4(float4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float4' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float4(float4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float4' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts a float8 into a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float8(float8) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float8' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float8(float8) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float8' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts a numeric into a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_numeric(numeric) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_numeric' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_numeric(numeric) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_numeric' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts a float8 into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8arr(float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8arr' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8arr(float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8arr' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an array of int8 positions, float8 values into an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_positions_float8arr(int8[],float8[],int8,float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_positions_float8arr' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_positions_float8arr(int8[],float8[],int8,float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_positions_float8arr' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Casts an SVEC into a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_return_array(MADLIB_SCHEMA.svec) RETURNS float8[] AS 'MODULE_PATHNAME', 'svec_return_array' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_return_array(MADLIB_SCHEMA.svec) RETURNS float8[] AS 'MODULE_PATHNAME', 'svec_return_array' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Concatenates two SVECs.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Replicates n copies of an SVEC and concatenates them together.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat_replicate(int4,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat_replicate' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat_replicate(int4,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat_replicate' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
@@ -1108,37 +1108,37 @@ CREATE OPERATOR MADLIB_SCHEMA.= (
-- Comparisons based on L2 Norm
--! Returns true if the l2 norm of the first SVEC is less than that of the second SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_lt' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_lt' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the l2 norm of the first SVEC is less than or equal to that of the second SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_le' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_le' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the l2 norm of the first SVEC is equal to that of the second SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_eq' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_eq' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the l2 norm of the first SVEC is not equal to that of the second SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ne' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ne' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the l2 norm of the first SVEC is greater than that of the second SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_gt' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_gt' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if the l2 norm of the first SVEC is greater than or equal to that of the second SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ge' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ge' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns a value indicating the relative values of the l2 norms of two SVECs.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_l2_cmp' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_l2_cmp' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
diff --git a/methods/svec_util/src/pg_gp/generate_svec.c b/methods/svec_util/src/pg_gp/generate_svec.cpp
similarity index 95%
rename from methods/svec_util/src/pg_gp/generate_svec.c
rename to methods/svec_util/src/pg_gp/generate_svec.cpp
index 1d7012c..b081b82 100644
--- a/methods/svec_util/src/pg_gp/generate_svec.c
+++ b/methods/svec_util/src/pg_gp/generate_svec.cpp
@@ -9,6 +9,14 @@
#include "../../../svec/src/pg_gp/sparse_vector.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum generate_sparse_vector(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
PG_FUNCTION_INFO_V1(generate_sparse_vector);
Datum generate_sparse_vector(PG_FUNCTION_ARGS)
{
diff --git a/methods/svec_util/src/pg_gp/gp_sfv.c b/methods/svec_util/src/pg_gp/gp_sfv.cpp
similarity index 99%
rename from methods/svec_util/src/pg_gp/gp_sfv.c
rename to methods/svec_util/src/pg_gp/gp_sfv.cpp
index 7b2aeb6..1cbdb1b 100644
--- a/methods/svec_util/src/pg_gp/gp_sfv.c
+++ b/methods/svec_util/src/pg_gp/gp_sfv.cpp
@@ -17,7 +17,13 @@
#include "../../../svec/src/pg_gp/sparse_vector.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
Datum gp_extract_feature_histogram(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
static void gp_extract_feature_histogram_errout(char *msg);
diff --git a/methods/svec_util/src/pg_gp/svec_util.c b/methods/svec_util/src/pg_gp/svec_util.cpp
similarity index 95%
rename from methods/svec_util/src/pg_gp/svec_util.c
rename to methods/svec_util/src/pg_gp/svec_util.cpp
index 12c839c..23e43d8 100644
--- a/methods/svec_util/src/pg_gp/svec_util.c
+++ b/methods/svec_util/src/pg_gp/svec_util.cpp
@@ -24,11 +24,54 @@
PG_MODULE_MAGIC;
#endif
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum float8_min(PG_FUNCTION_ARGS);
+Datum float8_max(PG_FUNCTION_ARGS);
+Datum svec_to_string(PG_FUNCTION_ARGS);
+Datum svec_from_string(PG_FUNCTION_ARGS);
+Datum svec_unnest(PG_FUNCTION_ARGS);
+Datum svec_dimension(PG_FUNCTION_ARGS);
+Datum svec_lapply(PG_FUNCTION_ARGS);
+Datum svec_append(PG_FUNCTION_ARGS);
+Datum svec_proj(PG_FUNCTION_ARGS);
+Datum svec_subvec(PG_FUNCTION_ARGS);
+Datum svec_reverse(PG_FUNCTION_ARGS);
+Datum svec_change(PG_FUNCTION_ARGS);
+Datum svec_eq_non_zero(PG_FUNCTION_ARGS);
+Datum svec_contains(PG_FUNCTION_ARGS);
+Datum svec_count(PG_FUNCTION_ARGS);
+Datum svec_l2norm(PG_FUNCTION_ARGS);
+Datum svec_svec_l2norm(PG_FUNCTION_ARGS);
+Datum svec_svec_l1norm(PG_FUNCTION_ARGS);
+Datum svec_svec_angle(PG_FUNCTION_ARGS);
+Datum svec_svec_tanimoto_distance(PG_FUNCTION_ARGS);
+Datum svec_normalize(PG_FUNCTION_ARGS);
+Datum svec_l1norm(PG_FUNCTION_ARGS);
+Datum svec_summate(PG_FUNCTION_ARGS);
+Datum svec_log(PG_FUNCTION_ARGS);
+Datum float8arr_l1norm(PG_FUNCTION_ARGS);
+Datum float8arr_summate(PG_FUNCTION_ARGS);
+Datum float8arr_l2norm(PG_FUNCTION_ARGS);
+Datum float8arr_hash(PG_FUNCTION_ARGS);
+Datum svec_pivot(PG_FUNCTION_ARGS);
+Datum float8arr_median(PG_FUNCTION_ARGS);
+Datum svec_median(PG_FUNCTION_ARGS);
+Datum svec_nonbase_positions(PG_FUNCTION_ARGS);
+Datum svec_nonbase_values(PG_FUNCTION_ARGS);
+Datum svec_hash(PG_FUNCTION_ARGS);
+Datum svec_mean_transition(PG_FUNCTION_ARGS);
+Datum svec_mean_prefunc(PG_FUNCTION_ARGS);
+Datum svec_mean_final(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
/*
* Basic floating point operators like MIN,MAX
*/
PG_FUNCTION_INFO_V1( float8_min );
-Datum float8_min(PG_FUNCTION_ARGS);
Datum float8_min(PG_FUNCTION_ARGS)
{
if (PG_ARGISNULL(0)) {
@@ -137,7 +180,7 @@ Datum svec_unnest(PG_FUNCTION_ARGS)
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
- fctx = funcctx->user_fctx;
+ fctx = (svec_unnest_fctx *) funcctx->user_fctx;
run_length = compword_to_int8(fctx->index_position);
@@ -573,7 +616,7 @@ Datum svec_normalize(PG_FUNCTION_ARGS)
norm = l2norm_sdata_values_double(sdata);
- op_sdata_by_scalar_inplace( 3, (char *)&norm, sdata, 2);
+ op_sdata_by_scalar_inplace( (operation_t)3, (char *)&norm, sdata, 2);
PG_RETURN_SVECTYPE_P( svec_from_sparsedata( sdata, true));
}
@@ -915,7 +958,7 @@ partition_select (char **lists, size_t nlists, size_t *widths,
tmp = (char **)palloc(nlists*sizeof(char *));
for (uint64 i=0;i<nlists;i++)
{
- tmp[i] = (void *)palloc(widths[i]);
+ tmp[i] = (char *)palloc(widths[i]);
}
pvalue = (char *)palloc(widths[0]);
diff --git a/methods/svec_util/src/pg_gp/svec_util.h b/methods/svec_util/src/pg_gp/svec_util.h
index 7c414e7..8e1d1d6 100644
--- a/methods/svec_util/src/pg_gp/svec_util.h
+++ b/methods/svec_util/src/pg_gp/svec_util.h
@@ -1,6 +1,10 @@
#ifndef SVECUTIL_H
#define SVECUTIL_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
Datum svec_l2norm(PG_FUNCTION_ARGS);
Datum svec_count(PG_FUNCTION_ARGS);
Datum svec_log(PG_FUNCTION_ARGS);
@@ -13,4 +17,8 @@ Datum svec_svec_l2norm(PG_FUNCTION_ARGS);
Datum svec_svec_angle(PG_FUNCTION_ARGS);
Datum svec_svec_tanimoto_distance(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
#endif
diff --git a/methods/svec_util/src/pg_gp/svec_util.sql_in b/methods/svec_util/src/pg_gp/svec_util.sql_in
index 31f8845..f7eeb61 100644
--- a/methods/svec_util/src/pg_gp/svec_util.sql_in
+++ b/methods/svec_util/src/pg_gp/svec_util.sql_in
@@ -18,7 +18,7 @@ m4_include(`SQLCommon.m4')
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gen_svec(BIGINT[], FLOAT8[], BIGINT)
RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'generate_sparse_vector'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
--! Creates the output table containing the sparse vector representation for the documents
@@ -48,163 +48,163 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
--! Basic floating point scalar operator: MIN.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dmin(float8,float8) RETURNS float8 AS 'MODULE_PATHNAME', 'float8_min' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dmin(float8,float8) RETURNS float8 AS 'MODULE_PATHNAME', 'float8_min' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Basic floating point scalar operator: MAX.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dmax(float8,float8) RETURNS float8 AS 'MODULE_PATHNAME', 'float8_max' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dmax(float8,float8) RETURNS float8 AS 'MODULE_PATHNAME', 'float8_max' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Counts the number of non-zero entries in the input vector; the second argument is capped at 1, then added to the first; used as the sfunc in the svec_count_nonzero() aggregate below.
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_count(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec
-AS 'MODULE_PATHNAME', 'svec_count' STRICT LANGUAGE C IMMUTABLE
+AS 'MODULE_PATHNAME', 'svec_count' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the logarithm of each element of the input SVEC.
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_log(MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_log' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_log(MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_log' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if two SVECs are equal, not counting zeros (zero equals anything). If the two SVECs are of different size, then the function essentially zero-pads the shorter one and performs the comparison.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_eq_non_zero(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_eq_non_zero' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_eq_non_zero(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_eq_non_zero' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns true if left svec contains right one, meaning that every non-zero value in the right svec equals left one
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_contains(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_contains' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_contains(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_contains' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the l2norm of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2norm(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_l2norm' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2norm(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_l2norm' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the l2norm of a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2norm(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_l2norm' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2norm(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_l2norm' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the l2norm distance between two SVECs.
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.l2norm(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec)
-RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_l2norm' LANGUAGE C STRICT IMMUTABLE
+RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_l2norm' LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the l1norm distance between two SVECs.
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.l1norm(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec)
-RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_l1norm' LANGUAGE C STRICT IMMUTABLE
+RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_l1norm' LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the l1norm of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l1norm(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_l1norm' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l1norm(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_l1norm' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the l1norm of a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l1norm(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_l1norm' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l1norm(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_l1norm' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the angle between two SVECs in radians.
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.angle(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec)
-RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_angle' LANGUAGE C STRICT IMMUTABLE
+RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_angle' LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the Tanimoto distance between two SVECs.
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tanimoto_distance(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec)
-RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_tanimoto_distance' LANGUAGE C STRICT IMMUTABLE
+RETURNS float8 AS 'MODULE_PATHNAME', 'svec_svec_tanimoto_distance' LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Unnests an SVEC into a table of uncompressed values
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_unnest(MADLIB_SCHEMA.svec) RETURNS setof float8 AS 'MODULE_PATHNAME', 'svec_unnest' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_unnest(MADLIB_SCHEMA.svec) RETURNS setof float8 AS 'MODULE_PATHNAME', 'svec_unnest' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Appends an element to the back of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_pivot(MADLIB_SCHEMA.svec,float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_pivot' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_pivot(MADLIB_SCHEMA.svec,float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_pivot' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Sums the elements of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_elsum(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_summate' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_elsum(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_summate' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Sums the elements of a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_elsum(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_summate' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_elsum(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_summate' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the median element of a float8 array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_median(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_median' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_median(float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_median' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the median element of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_median(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_median' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_median(MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_median' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Compares an SVEC to a float8, and returns positions of all elements not equal to the float as an array. Element index here starts at 0.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_nonbase_positions(MADLIB_SCHEMA.svec, FLOAT8) RETURNS INT8[] AS 'MODULE_PATHNAME', 'svec_nonbase_positions' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_nonbase_positions(MADLIB_SCHEMA.svec, FLOAT8) RETURNS INT8[] AS 'MODULE_PATHNAME', 'svec_nonbase_positions' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Compares an SVEC to a float8, and returns values of all elements not equal to the float as an array.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_nonbase_values(MADLIB_SCHEMA.svec, FLOAT8) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svec_nonbase_values' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_nonbase_values(MADLIB_SCHEMA.svec, FLOAT8) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svec_nonbase_values' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Returns the dimension of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dimension(MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_dimension' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dimension(MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_dimension' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Applies a given function to each element of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_lapply(text,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_lapply' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_lapply(text,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_lapply' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Appends a run-length block to the back of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_append(MADLIB_SCHEMA.svec,float8,int8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_append' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_append(MADLIB_SCHEMA.svec,float8,int8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_append' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Projects onto an element of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_proj(MADLIB_SCHEMA.svec,int4) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_proj' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_proj(MADLIB_SCHEMA.svec,int4) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_proj' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Extracts a subvector of an SVEC given the subvector's start and end indices.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_subvec(MADLIB_SCHEMA.svec,int4,int4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_subvec' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_subvec(MADLIB_SCHEMA.svec,int4,int4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_subvec' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Reverses the elements of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_reverse(MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_reverse' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_reverse(MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_reverse' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Replaces the subvector of a given SVEC at a given start index with another SVEC. Note that element index should start at 1.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_change(MADLIB_SCHEMA.svec,int4,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_change' LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_change(MADLIB_SCHEMA.svec,int4,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_change' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the hash of an SVEC.
--!
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_hash(MADLIB_SCHEMA.svec) RETURNS int4 AS 'MODULE_PATHNAME', 'svec_hash' STRICT LANGUAGE C IMMUTABLE
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_hash(MADLIB_SCHEMA.svec) RETURNS int4 AS 'MODULE_PATHNAME', 'svec_hash' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Computes the word-occurence vector of a document
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_sfv(text[], text[]) RETURNS MADLIB_SCHEMA.svec AS
-'MODULE_PATHNAME', 'gp_extract_feature_histogram' LANGUAGE C IMMUTABLE
+'MODULE_PATHNAME', 'gp_extract_feature_histogram' LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Sorts an array of texts. This function should be in MADlib common.
@@ -217,13 +217,13 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `READS SQL DATA', `');
--! Converts an svec to a text string
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_to_string(MADLIB_SCHEMA.svec)
-RETURNS text AS 'MODULE_PATHNAME', 'svec_to_string' STRICT LANGUAGE C IMMUTABLE
+RETURNS text AS 'MODULE_PATHNAME', 'svec_to_string' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Converts a text string to an svec
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_from_string(text)
-RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_from_string' STRICT LANGUAGE C IMMUTABLE
+RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_from_string' STRICT LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -231,21 +231,21 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mean_transition( FLOAT[], MADLIB_SCHEMA.svec)
RETURNS FLOAT[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Preliminary merge function for mean(svec) aggregate
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mean_prefunc( FLOAT[], FLOAT[])
RETURNS FLOAT[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Final function for mean(svec) aggregate
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mean_final( FLOAT[])
RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--! Aggregate that computes the element-wise mean of a list of vectors.
@@ -303,5 +303,5 @@ CREATE AGGREGATE MADLIB_SCHEMA.svec_median_inmemory (float8) (
--! Normalizes an SVEC that is divides all elements by its norm/magnitude.
--!
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalize(MADLIB_SCHEMA.svec)
-RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_normalize' LANGUAGE C IMMUTABLE STRICT
+RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_normalize' LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/methods/utils/src/pg_gp/exec_sql_using.c b/methods/utils/src/pg_gp/exec_sql_using.cpp
similarity index 97%
rename from methods/utils/src/pg_gp/exec_sql_using.c
rename to methods/utils/src/pg_gp/exec_sql_using.cpp
index 268a143..78891bc 100644
--- a/methods/utils/src/pg_gp/exec_sql_using.c
+++ b/methods/utils/src/pg_gp/exec_sql_using.cpp
@@ -22,6 +22,14 @@
#include <executor/spi.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum exec_sql_using(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
// SearchSysCache1 first appeared with commit e26c539 by Robert Haas
// <rhaas@postgresql.org> on Sun, 14 Feb 2010 18:42:19 +0000. If this macro is
// not defined, we assume to be on a version of PostgreSQL that expects five
@@ -105,7 +113,7 @@ exec_sql_using(PG_FUNCTION_ARGS) {
for (int i = 1; i < nargs; i++){
if (PG_ARGISNULL(i)) {
if (nulls == NULL) {
- nulls = palloc0(sizeof(char) * (nargs - 1));
+ nulls = (char *) palloc0(sizeof(char) * (nargs - 1));
memset(nulls, ' ', nargs - 1);
}
nulls[i - 1] = 'n';
diff --git a/methods/utils/src/pg_gp/noop.c b/methods/utils/src/pg_gp/noop.cpp
similarity index 59%
rename from methods/utils/src/pg_gp/noop.c
rename to methods/utils/src/pg_gp/noop.cpp
index 5521cad..7ac77cd 100644
--- a/methods/utils/src/pg_gp/noop.c
+++ b/methods/utils/src/pg_gp/noop.cpp
@@ -1,6 +1,14 @@
#include <postgres.h>
#include <fmgr.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum noop(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
PG_FUNCTION_INFO_V1(noop);
Datum
noop(PG_FUNCTION_ARGS) {
diff --git a/methods/utils/src/pg_gp/to_char.c b/methods/utils/src/pg_gp/to_char.cpp
similarity index 89%
rename from methods/utils/src/pg_gp/to_char.c
rename to methods/utils/src/pg_gp/to_char.cpp
index 2d33676..2dc2f43 100644
--- a/methods/utils/src/pg_gp/to_char.c
+++ b/methods/utils/src/pg_gp/to_char.cpp
@@ -7,6 +7,14 @@
PG_MODULE_MAGIC;
#endif
+#ifdef __cplusplus
+extern "C" {
+#endif
+Datum __to_text(PG_FUNCTION_ARGS);
+#ifdef __cplusplus
+}
+#endif
+
PG_FUNCTION_INFO_V1(__to_text);
/*
* @brief Cast a value to text. On some databases, there
diff --git a/pom.xml b/pom.xml
index 4d0eec1..d487d18 100644
--- a/pom.xml
+++ b/pom.xml
@@ -122,20 +122,20 @@
<exclude>examples/gpce/sketches.sql</exclude>
<exclude>examples/gpce/svd-mf.sql</exclude>
<exclude>examples/gpce/svm.sql</exclude>
- <exclude>methods/array_ops/src/pg_gp/array_ops.c</exclude>
+ <exclude>methods/array_ops/src/pg_gp/array_ops.cpp</exclude>
<exclude>methods/array_ops/src/pg_gp/array_ops.sql_in</exclude>
<exclude>methods/array_ops/src/pg_gp/test/array_ops.sql_in</exclude>
- <exclude>methods/kmeans/src/pg_gp/kmeans.c</exclude>
+ <exclude>methods/kmeans/src/pg_gp/kmeans.cpp</exclude>
<exclude>methods/sketch/src/pg_gp/__init__.py_in</exclude>
- <exclude>methods/sketch/src/pg_gp/countmin.c</exclude>
+ <exclude>methods/sketch/src/pg_gp/countmin.cpp</exclude>
<exclude>methods/sketch/src/pg_gp/countmin.h</exclude>
<exclude>methods/sketch/src/pg_gp/countmin.py_in</exclude>
<exclude>methods/sketch/src/pg_gp/fm.c</exclude>
- <exclude>methods/sketch/src/pg_gp/mfvsketch.c</exclude>
+ <exclude>methods/sketch/src/pg_gp/mfvsketch.cpp</exclude>
<exclude>methods/sketch/src/pg_gp/sketch.sql_in</exclude>
- <exclude>methods/sketch/src/pg_gp/sketch_support.c</exclude>
+ <exclude>methods/sketch/src/pg_gp/sketch_support.cpp</exclude>
<exclude>methods/sketch/src/pg_gp/sketch_support.h</exclude>
- <exclude>methods/sketch/src/pg_gp/sortasort.c</exclude>
+ <exclude>methods/sketch/src/pg_gp/sortasort.cpp</exclude>
<exclude>methods/sketch/src/pg_gp/sortasort.h</exclude>
<exclude>methods/sketch/src/pg_gp/sql/cm.sql_in</exclude>
<exclude>methods/sketch/src/pg_gp/sql/fm.sql_in</exclude>
@@ -143,25 +143,25 @@
<exclude>methods/sketch/src/pg_gp/sql/support.sql_in</exclude>
<exclude>methods/sketch/src/pg_gp/uncrustify.cfg</exclude>
<exclude>methods/svec/src/pg_gp/float_specials.h</exclude>
- <exclude>methods/svec/src/pg_gp/operators.c</exclude>
- <exclude>methods/svec/src/pg_gp/sparse_vector.c</exclude>
+ <exclude>methods/svec/src/pg_gp/operators.cpp</exclude>
+ <exclude>methods/svec/src/pg_gp/sparse_vector.cpp</exclude>
<exclude>methods/svec/src/pg_gp/sparse_vector.h</exclude>
- <exclude>methods/svec/src/pg_gp/SparseData.c</exclude>
+ <exclude>methods/svec/src/pg_gp/SparseData.cpp</exclude>
<exclude>methods/svec/src/pg_gp/SparseData.h</exclude>
<exclude>methods/svec/src/pg_gp/svec.sql_in</exclude>
<exclude>methods/svec_util/src/pg_gp/__init__.py_in</exclude>
- <exclude>methods/svec_util/src/pg_gp/generate_svec.c</exclude>
+ <exclude>methods/svec_util/src/pg_gp/generate_svec.cpp</exclude>
<exclude>methods/svec_util/src/pg_gp/generate_svec.py_in</exclude>
- <exclude>methods/svec_util/src/pg_gp/gp_sfv.c</exclude>
+ <exclude>methods/svec_util/src/pg_gp/gp_sfv.cpp</exclude>
<exclude>methods/svec_util/src/pg_gp/sql/gp_sfv.sql_in.todo.sql</exclude>
<exclude>methods/svec_util/src/pg_gp/sql/gp_sfv_sort_order.sql_in</exclude>
<exclude>methods/svec_util/src/pg_gp/sql/svec_test.sql_in</exclude>
- <exclude>methods/svec_util/src/pg_gp/svec_util.c</exclude>
+ <exclude>methods/svec_util/src/pg_gp/svec_util.cpp</exclude>
<exclude>methods/svec_util/src/pg_gp/svec_util.h</exclude>
<exclude>methods/svec_util/src/pg_gp/svec_util.sql_in</exclude>
- <exclude>methods/utils/src/pg_gp/exec_sql_using.c</exclude>
- <exclude>methods/utils/src/pg_gp/noop.c</exclude>
- <exclude>methods/utils/src/pg_gp/to_char.c</exclude>
+ <exclude>methods/utils/src/pg_gp/exec_sql_using.cpp</exclude>
+ <exclude>methods/utils/src/pg_gp/noop.cpp</exclude>
+ <exclude>methods/utils/src/pg_gp/to_char.cpp</exclude>
<exclude>src/bin/CMakeLists.txt</exclude>
<exclude>src/bin/madpack</exclude>
<exclude>src/CMakeLists.txt</exclude>
diff --git a/src/config/Modules.yml b/src/config/Modules.yml
index f2da10f..c1e5004 100644
--- a/src/config/Modules.yml
+++ b/src/config/Modules.yml
@@ -11,8 +11,8 @@ modules:
- name: convex
depends: ['utilities']
- name: crf
- - name: deep_learning
- depends: ['utilities']
+# - name: deep_learning
+# depends: ['utilities']
- name: elastic_net
- name: glm
depends: ['utilities']
@@ -29,7 +29,7 @@ modules:
depends: ['utilities']
- name: pca
depends: ['linalg', 'convex']
- - name: pmml
+# - name: pmml
- name: prob
- name: recursive_partitioning
depends: ['utilities']
@@ -52,3 +52,9 @@ modules:
- name: validation
depends: ['array_ops', 'regress']
- name: stemmer
+ - name: xgboost_gs
+ depends: ['utilities']
+ - name: facebook_prophet
+ depends: ['utilities']
+ - name: gbdt
+ depends: ['utilities', 'recursive_partitioning']
diff --git a/src/config/Ports.yml b/src/config/Ports.yml
index 78327d7..6502c96 100644
--- a/src/config/Ports.yml
+++ b/src/config/Ports.yml
@@ -9,3 +9,6 @@ postgres:
greenplum:
name: Greenplum DB
+
+opengauss:
+ name: openGauss DB
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index 2af0480..68a6566 100644
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -24,7 +24,7 @@ from utilities import is_rev_gte
from utilities import remove_comments_from_sql
from utilities import run_query
# Required Python version
-py_min_ver = [2, 6]
+py_min_ver = [2, 7]
# Check python version
if sys.version_info[:2] < py_min_ver:
@@ -59,7 +59,7 @@ portid_list = []
for port in ports:
portid_list.append(port)
-SUPPORTED_PORTS = ('postgres', 'greenplum')
+SUPPORTED_PORTS = ('postgres', 'greenplum', 'opengauss')
# Global variables
portid = None # Target port ID (eg: pg90, gp40)
@@ -110,7 +110,7 @@ def _get_relative_maddir(maddir, port):
madlib function definition. Replacing the GPHOME path with the equivalent
relative path makes it simpler to perform DB upgrades without breaking MADlib.
"""
- if port == 'postgres':
+ if port == 'postgres' or port == 'opengauss':
# do nothing for postgres
return maddir
@@ -167,18 +167,22 @@ def _run_m4_and_append(schema, maddir_mod_py, module, sqlfile,
if pre_sql:
output_filehandle.writelines([pre_sql, '\n\n'])
# Find the madpack dir (platform specific or generic)
+ global portid
+ portid_old = portid
+ portid = 'postgres' if portid == 'opengauss' else portid
if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/madpack"):
maddir_madpack = maddir + "/ports/" + portid + "/" + dbver + "/madpack"
else:
maddir_madpack = maddir + "/madpack"
maddir_ext_py = maddir + "/lib/python"
+ portid = portid_old
m4args = ['m4',
'-P',
'-DMADLIB_SCHEMA=' + schema,
'-DPLPYTHON_LIBDIR=' + maddir_mod_py,
'-DEXT_PYTHON_LIBDIR=' + maddir_ext_py,
- '-DMODULE_PATHNAME=' + maddir_lib,
+ '-DMODULE_PATHNAME=' + '$libdir/libmadlib.so',
'-DMODULE_NAME=' + module,
'-I' + maddir_madpack,
sqlfile]
@@ -213,7 +217,7 @@ def _run_install_check_sql(schema, maddir_mod_py, module, sqlfile,
sub_module = ''
# Run the SQL using DB command-line utility
if portid in SUPPORTED_PORTS:
- sqlcmd = 'psql'
+ sqlcmd = 'gsql'
# Test the DB cmd line utility
std, err = subprocess.Popen(['which', sqlcmd], stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
@@ -226,12 +230,8 @@ def _run_install_check_sql(schema, maddir_mod_py, module, sqlfile,
'-p', con_args['host'].split(':')[1],
'-d', con_args['database'],
'-U', con_args['user'],
- '--no-password',
+ '-W', con_args['password'],
'-f', tmpfile]
- runenv = os.environ
- if 'password' in con_args:
- runenv["PGPASSWORD"] = con_args['password']
- runenv["PGOPTIONS"] = '-c client_min_messages=notice'
# Open log file
try:
@@ -243,7 +243,7 @@ def _run_install_check_sql(schema, maddir_mod_py, module, sqlfile,
# Run the SQL
try:
info_(this, "> ... executing " + tmpfile, verbose)
- retval = subprocess.call(runcmd, env=runenv, stdout=log, stderr=log)
+ retval = subprocess.call(runcmd, stdout=log, stderr=log)
except:
error_(this, "Failed executing %s" % tmpfile, False)
raise Exception
@@ -261,7 +261,7 @@ def _run_sql_file(schema, sqlfile):
"""
# Run the SQL using DB command-line utility
if portid in SUPPORTED_PORTS:
- sqlcmd = 'psql'
+ sqlcmd = 'gsql'
# Test the DB cmd line utility
std, err = subprocess.Popen(['which', sqlcmd], stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
@@ -274,13 +274,9 @@ def _run_sql_file(schema, sqlfile):
'-p', con_args['host'].split(':')[1],
'-d', con_args['database'],
'-U', con_args['user'],
- '--no-password',
- '--single-transaction',
+ '-W', con_args['password'],
+ #'--single-transaction',
'-f', sqlfile]
- runenv = os.environ
- if 'password' in con_args:
- runenv["PGPASSWORD"] = con_args['password']
- runenv["PGOPTIONS"] = '-c client_min_messages=notice'
# Open log file
logfile = sqlfile + '.log'
@@ -294,7 +290,7 @@ def _run_sql_file(schema, sqlfile):
try:
info_(this, "> ... executing " + sqlfile, verbose)
info_(this, ' '.join(runcmd), verbose)
- retval = subprocess.call(runcmd, env=runenv, stdout=log, stderr=log)
+ retval = subprocess.call(runcmd, stdout=log, stderr=log)
except:
error_(this, "Failed executing %s" % sqlfile, False)
raise Exception
@@ -353,6 +349,12 @@ def _check_db_port(portid):
return True
elif portid == 'greenplum':
return True
+ if row and row[0]['version'].lower().find('opengauss') >= 0:
+ if portid == 'opengauss':
+ return True
+ if row and row[0]['version'].find('GaussDB') >= 0:
+ if portid == 'opengauss':
+ return True
return False
# ------------------------------------------------------------------------------
@@ -469,6 +471,8 @@ def _db_upgrade(schema, filehandle, db_madlib_ver):
info_(this, "\tDetecting dependencies...", True)
info_(this, "\tLoading change list...", True)
+ portid_old = portid
+ portid = 'postgres' if portid == 'opengauss' else portid
ch = uu.ChangeHandler(schema, portid, con_args, maddir, db_madlib_ver, filehandle)
info_(this, "\tDetecting table dependencies...", True)
@@ -476,7 +480,7 @@ def _db_upgrade(schema, filehandle, db_madlib_ver):
info_(this, "\tDetecting view dependencies...", True)
vd = uu.ViewDependency(schema, portid, con_args)
-
+ portid = portid_old
abort = False
if td.has_dependency():
info_(this, "*" * 50, True)
@@ -562,7 +566,10 @@ def _db_upgrade(schema, filehandle, db_madlib_ver):
info_(this, "No dependency problem found, continuing to upgrade ...", True)
info_(this, "\tReading existing UDAs/UDTs...", False)
+ portid_old = portid
+ portid = 'postgres' if portid == 'opengauss' else portid
sc = uu.ScriptCleaner(schema, portid, con_args, ch)
+ portid = portid_old
info_(this, "Script Cleaner initialized ...", False)
ch.drop_changed_uda()
@@ -632,6 +639,7 @@ def _process_py_sql_files_in_modules(modset, args_dict):
error_(this, "Invalid madpack operation: %s" % madpack_cmd, True)
# Perform operations on all modules
+ portid_new = 'postgres' if portid == 'opengauss' else portid
for moduleinfo in portspecs['modules']:
# Get the module name
module = moduleinfo['name']
@@ -641,14 +649,14 @@ def _process_py_sql_files_in_modules(modset, args_dict):
continue
# Find the Python module dir (platform specific or generic)
- if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/modules/" + module):
- maddir_mod_py = maddir + "/ports/" + portid + "/" + dbver + "/modules"
+ if os.path.isdir(maddir + "/ports/" + portid_new + "/" + dbver + "/modules/" + module):
+ maddir_mod_py = maddir + "/ports/" + portid_new + "/" + dbver + "/modules"
else:
maddir_mod_py = maddir + "/modules"
# Find the SQL module dir (platform specific or generic)
- if os.path.isdir(maddir + "/ports/" + portid + "/modules/" + module):
- maddir_mod_sql = maddir + "/ports/" + portid + "/modules"
+ if os.path.isdir(maddir + "/ports/" + portid_new + "/modules/" + module):
+ maddir_mod_sql = maddir + "/ports/" + portid_new + "/modules"
elif os.path.isdir(maddir + "/modules/" + module):
maddir_mod_sql = maddir + "/modules"
else:
@@ -757,8 +765,7 @@ def _execute_per_module_unit_test_algo(module, pyfile, cur_tmpdir):
run_start = datetime.datetime.now()
# Run the python unit test file
runcmd = ["python", pyfile]
- runenv = os.environ
- retval = subprocess.call(runcmd, env=runenv, stdout=log, stderr=log)
+ retval = subprocess.call(runcmd, stdout=log, stderr=log)
run_end = datetime.datetime.now()
milliseconds = round((run_end - run_start).seconds * 1000 +
(run_end - run_start).microseconds / 1000)
@@ -783,15 +790,16 @@ def _execute_per_module_install_dev_check_algo(schema, test_user,
test_schema = "madlib_installcheck_%s" % (module)
_internal_run_query("DROP SCHEMA IF EXISTS %s CASCADE; CREATE SCHEMA %s;" %
(test_schema, test_schema), True)
- _internal_run_query("GRANT ALL ON SCHEMA %s TO \"%s\";" %
+ _internal_run_query("GRANT ALL ON SCHEMA %s TO %s;" %
(test_schema, test_user), True)
# Switch to test user and prepare the search_path
pre_sql = '-- Switch to test user:\n' \
- 'SET ROLE \"%s\";\n' \
+ "SET ROLE %s password '%s';\n" \
'-- Set SEARCH_PATH for install-check:\n' \
'SET search_path=%s,%s;\n' \
- % (test_user, test_schema, schema)
+ 'SET behavior_compat_options = \'bind_procedure_searchpath\';\n' \
+ % (test_user, con_args['password'], test_schema, schema)
# Set file names
@@ -1031,9 +1039,10 @@ def run_install_check(args, testcase, madpack_cmd):
_internal_run_query("DROP OWNED BY \"%s\" CASCADE;" % (test_user), True)
_internal_run_query("DROP USER IF EXISTS \"%s\";" % (test_user), True)
- _internal_run_query("CREATE USER \"%s\" WITH SUPERUSER NOINHERIT;" % (test_user), True)
- _internal_run_query("GRANT USAGE ON SCHEMA %s TO \"%s\";" % (schema, test_user), True)
- _internal_run_query("GRANT ALL PRIVILEGES ON DATABASE \"%s\" TO \"%s\";" % (db_name, test_user), True)
+ _internal_run_query("CREATE USER %s WITH SYSADMIN NOINHERIT password '%s';" % (test_user, con_args['password']), True)
+
+ _internal_run_query("GRANT USAGE ON SCHEMA %s TO %s;" % (schema, test_user), True)
+ _internal_run_query("GRANT ALL PRIVILEGES ON DATABASE %s TO %s;" % (db_name, test_user), True)
# 2) Run test SQLs
info_(this, "> Running %s scripts for:" % madpack_cmd, verbose)
@@ -1262,8 +1271,7 @@ def main(argv):
con_args['host'] = c_host + ':' + c_port
con_args['database'] = c_db
con_args['user'] = c_user
- if c_pass is not None:
- con_args['password'] = c_pass
+ con_args['password'] = getpass.getpass("Password for user %s: " % c_user)
# Try connecting to the database
info_(this, "Testing database connection...", verbose)
@@ -1288,7 +1296,7 @@ def main(argv):
# Get MADlib version in DB
db_madlib_ver = get_db_madlib_version(con_args, schema)
- portdir = os.path.join(maddir, "ports", portid)
+ portdir = os.path.join(maddir, "ports", 'postgres' if portid == 'opengauss' else portid)
supportedVersions = [dirItem for dirItem in os.listdir(portdir)
if os.path.isdir(os.path.join(portdir, dirItem)) and
re.match("^\d+", dirItem)]
@@ -1319,7 +1327,7 @@ def main(argv):
else:
# only need the first two digits for <= 4.3.4
dbver = '.'.join(map(str, dbver_split[:2]))
- elif portid == 'postgres':
+ elif portid == 'postgres' or portid == 'opengauss':
if is_rev_gte(dbver_split, get_rev_num('10.0')):
# Postgres starting 10.0 uses semantic versioning. Hence,
# only need first digit for major version.
@@ -1335,6 +1343,8 @@ def main(argv):
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
+ portid_old = portid
+ portid = 'postgres' if portid == 'opengauss' else portid
if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/config"):
maddir_conf = maddir + "/ports/" + portid + "/" + dbver + "/config"
else:
@@ -1347,7 +1357,7 @@ def main(argv):
"/lib/libmadlib.so"
else:
maddir_lib = maddir + "/lib/libmadlib.so"
-
+ portid = portid_old
# Get the list of modules for this port
global portspecs
portspecs = configyml.get_modules(maddir_conf)
diff --git a/src/madpack/utilities.py b/src/madpack/utilities.py
index 9e4e95b..304f7a3 100644
--- a/src/madpack/utilities.py
+++ b/src/madpack/utilities.py
@@ -83,7 +83,7 @@ def remove_comments_from_sql(sql):
def run_query(sql, con_args, show_error=True):
# Define sqlcmd
- sqlcmd = 'psql'
+ sqlcmd = 'gsql'
delimiter = ' <$madlib_delimiter$> '
# Test the DB cmd line utility
@@ -98,10 +98,7 @@ def run_query(sql, con_args, show_error=True):
'-p', con_args['host'].split(':')[1],
'-d', con_args['database'],
'-U', con_args['user'],
- '-F', delimiter,
- '--no-password',
- '--no-psqlrc',
- '--no-align',
+ '-W', con_args['password'],
'-c', sql]
runenv = os.environ
if 'password' in con_args:
@@ -123,22 +120,23 @@ def run_query(sql, con_args, show_error=True):
i = 0
for line in std.splitlines():
if i == 0:
- cols = [name for name in line.split(delimiter)]
+ cols = [name.strip() for name in line.split(delimiter)]
else:
row = {} # dict of col_name:col_value pairs
c = 0
for val in line.split(delimiter):
- row[cols[c]] = val
+ row[cols[c]] = val.strip()
c += 1
results.insert(i, row)
i += 1
- # Drop the last line: "(X rows)"
+ # Drop the last 2 lines: "(X rows), ("\n")"
try:
results.pop()
+ results.pop()
except Exception:
pass
- return results
+ return results[1:]
# ------------------------------------------------------------------------------
@@ -176,7 +174,11 @@ def get_db_madlib_version(con_args, schema):
def get_dbver(con_args, portid):
""" Read version number from database (of form X.Y) """
try:
- versionStr = run_query("SELECT pg_catalog.version()", con_args, True)[0]['version']
+ versionStr = run_query("SELECT pg_catalog.version()", con_args, True)
+ versionStr = versionStr[0]['version']
+ if portid == 'opengauss' and ('openGauss' in versionStr or 'GaussDB' in versionStr):
+ return '9.2'
+
if portid == 'postgres':
match = re.search("PostgreSQL[a-zA-Z\s]*(\d+\.\d+)", versionStr)
elif portid == 'greenplum':
diff --git a/src/ports/CMakeLists.txt b/src/ports/CMakeLists.txt
index bb08e83..f7ec404 100644
--- a/src/ports/CMakeLists.txt
+++ b/src/ports/CMakeLists.txt
@@ -1,2 +1,2 @@
add_subdirectory(postgres)
-add_subdirectory(greenplum)
+#add_subdirectory(greenplum)
diff --git a/src/ports/postgres/CMakeLists.txt b/src/ports/postgres/CMakeLists.txt
index 6c7b3a8..6919024 100644
--- a/src/ports/postgres/CMakeLists.txt
+++ b/src/ports/postgres/CMakeLists.txt
@@ -59,7 +59,7 @@ list(APPEND MAD_DBAL_SOURCES
# BEGIN Legacy Code
file(GLOB_RECURSE LEGACY_C_FILES
- "${CMAKE_SOURCE_DIR}/methods/*.c")
+ "${CMAKE_SOURCE_DIR}/methods/*.cpp")
list(APPEND MAD_DBAL_SOURCES ${LEGACY_C_FILES})
# END Legacy Code
@@ -154,7 +154,7 @@ function(add_${PORT_LC}_library IN_PORT_VERSION)
# The warning GCC would output without -fno-strict-aliasing is:
# dereferencing type-punned pointer will break strict-aliasing rules
set_source_files_properties(${LEGACY_C_FILES} PROPERTIES
- COMPILE_FLAGS "-fno-strict-aliasing")
+ COMPILE_FLAGS "-fno-strict-aliasing -fsigned-char")
endif(CMAKE_COMPILER_IS_GNUCC)
# errOmitLocation is used in legacy C code, but it is Greenplum only
@@ -226,6 +226,11 @@ function(add_${PORT_LC}_library IN_PORT_VERSION)
DESTINATION ports/${PORT_DIR_NAME}/${IN_PORT_VERSION}/madpack
COMPONENT ${DBMS}
)
+ # copy libmadlib.so to $GAUSSHOME/lib/postgresql
+ MESSAGE(STATUS "HOME dir: $ENV{GAUSSHOME}")
+ if(DEFINED ENV{GAUSSHOME})
+ install(TARGETS madlib_${DBMS} LIBRARY DESTINATION $ENV{GAUSSHOME}/lib/postgresql)
+ endif()
endfunction(add_${PORT_LC}_library)
diff --git a/src/ports/postgres/dbconnector/AnyType_impl.hpp b/src/ports/postgres/dbconnector/AnyType_impl.hpp
index 41fcd47..4f0011c 100644
--- a/src/ports/postgres/dbconnector/AnyType_impl.hpp
+++ b/src/ports/postgres/dbconnector/AnyType_impl.hpp
@@ -258,7 +258,7 @@ AnyType::numFields() const {
case Scalar: return 1;
case ReturnComposite: return static_cast<uint16_t>(mChildren.size());
case FunctionComposite: return PG_NARGS();
- case NativeComposite: return HeapTupleHeaderGetNatts(mTupleHeader);
+ case NativeComposite: return ((mTupleHeader)->t_infomask2 & HEAP_NATTS_MASK); //HeapTupleHeaderGetNatts1(mTupleHeader);
default:
// This should never happen
throw std::logic_error("Unhandled case in AnyType::numFields().");
diff --git a/src/ports/postgres/dbconnector/dbconnector.hpp b/src/ports/postgres/dbconnector/dbconnector.hpp
index 523fd1c..e6222e0 100644
--- a/src/ports/postgres/dbconnector/dbconnector.hpp
+++ b/src/ports/postgres/dbconnector/dbconnector.hpp
@@ -18,9 +18,7 @@
// (commit https://github.com/postgres/postgres/commit/a919937f112eb2f548d5f9bd1b3a7298375e6380)
// Since we don't need anything from ports.h we can cheat and say its already been declared.
// Warning: This could cause problems in the future...
-#define PG_PORT_H
-extern "C" {
#include <postgres.h>
#include <pg_config.h> // Use the macro defined in the header to detect the platform
#include <funcapi.h>
@@ -40,7 +38,6 @@ extern "C" {
#include <utils/syscache.h> // for direct access to catalog, e.g., SearchSysCache()
#include <utils/typcache.h> // type conversion, e.g., lookup_rowtype_tupdesc
#include "../../../../methods/svec/src/pg_gp/sparse_vector.h" // Legacy sparse vectors
-} // extern "C"
#include "Compatibility.hpp"
@@ -77,6 +74,10 @@ extern "C" {
#undef dngettext
#endif
+#ifdef PM
+#undef PM
+#endif
+
// Note: If errors occur in the following include files, it could indicate that
// new macros have been added to PostgreSQL header files.
#include <boost/mpl/if.hpp>
diff --git a/src/ports/postgres/dbconnector/main.cpp b/src/ports/postgres/dbconnector/main.cpp
index 1422780..160b11d 100644
--- a/src/ports/postgres/dbconnector/main.cpp
+++ b/src/ports/postgres/dbconnector/main.cpp
@@ -11,9 +11,7 @@
#include <dbconnector/dbconnector.hpp>
#include <utils/MallocAllocator.hpp>
-extern "C" {
- PG_MODULE_MAGIC;
-} // extern "C"
+PG_MODULE_MAGIC;
namespace madlib {
diff --git a/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in b/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in
index 6b7a7c9..e21268b 100644
--- a/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in
+++ b/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in
@@ -12,6 +12,7 @@ import plpy
from utilities.validate_args import columns_exist_in_table
from utilities.validate_args import table_exists
from utilities.control import MinWarning
+from utilities.utilities import unique_string
"""
@@ -144,7 +145,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
plpy.execute("""
CREATE TEMP TABLE assoc_rules_aux_tmp
(
- ruleId SERIAL,
+ ruleId BIGINT,
pre TEXT,
post TEXT,
support FLOAT8,
@@ -165,17 +166,17 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
plpy.execute("""
CREATE TEMP TABLE assoc_input_unique AS
SELECT
- dense_rank() OVER (ORDER BY tid)::BIGINT as tid,
+ dense_rank() OVER (ORDER BY artid)::BIGINT as artid,
item
FROM (
- SELECT {0} as tid,
+ SELECT {0} as artid,
{1}::text as item
FROM {2}
WHERE {3} IS NOT NULL AND
{4} IS NOT NULL
GROUP BY 1,2
) t
- m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (tid)')
+ m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (artid)')
""".format(tid_col, item_col, input_table, tid_col, item_col)
);
@@ -183,7 +184,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
plpy.info("finished removing duplicates");
rv = plpy.execute("""
- SELECT count(DISTINCT tid) as c1, count(DISTINCT item) as c2
+ SELECT count(DISTINCT artid) as c1, count(DISTINCT item) as c2
FROM assoc_input_unique
""");
num_tranx = rv[0]["c1"];
@@ -203,7 +204,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
item::TEXT,
cnt::FLOAT8
FROM (
- SELECT item AS item, count(tid) as cnt
+ SELECT item AS item, count(artid) as cnt
FROM assoc_input_unique
GROUP BY item
) t
@@ -219,11 +220,11 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
plpy.execute("DROP TABLE IF EXISTS assoc_enc_input");
plpy.execute("""
- CREATE TEMP TABLE assoc_enc_input (tid, item, cnt) AS
- SELECT t1.tid, t2.item_id, t2.cnt
+ CREATE TEMP TABLE assoc_enc_input (artid, item, cnt) AS
+ SELECT t1.artid, t2.item_id, t2.cnt
FROM assoc_input_unique t1, assoc_item_uniq t2
WHERE t1.item = t2.item_text
- m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (tid)')
+ m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (artid)')
""");
if verbose :
@@ -281,7 +282,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
(ARRAY[t.item], ARRAY[1], {1}, 0),
t.cnt::FLOAT8 / {2},
{3}.svec_cast_positions_float8arr
- (array_agg(t.tid), array_agg(1), {4}, 0)
+ (array_agg(t.artid), array_agg(1), {4}, 0)
FROM assoc_enc_input t
GROUP BY t.item, t.cnt
""".format(madlib_schema, num_supp_prod, num_tranx, madlib_schema,
@@ -314,7 +315,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
plpy.execute("""
CREATE TEMP TABLE assoc_loop_aux
(
- id BIGSERIAL,
+ id BIGINT,
set_list {0}.svec,
support FLOAT8,
tids {1}.svec
@@ -380,13 +381,15 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
# The iterp1 check ensures that the new itemset is of a certain size.
# At every iteration, we increase the target itemset size by one.
- plpy.execute("ALTER SEQUENCE assoc_loop_aux_id_seq RESTART WITH 1");
+ # plpy.execute("ALTER SEQUENCE assoc_loop_aux_id_seq RESTART WITH 1");
+ assoc_loop_aux_sequence = unique_string(desp='s1')
plpy.execute("TRUNCATE TABLE assoc_loop_aux");
+ plpy.execute("""CREATE SEQUENCE {s} START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1;
+ """.format(s=assoc_loop_aux_sequence))
plpy.execute("""
- INSERT INTO assoc_loop_aux(set_list, support, tids)
- SELECT DISTINCT ON({madlib_schema}.svec_to_string(set_list)) set_list,
- {madlib_schema}.svec_l1norm(tids)::FLOAT8 / {num_tranx},
- tids
+ INSERT INTO assoc_loop_aux(set_list, support, tids, id)
+ SELECT set_list, {madlib_schema}.svec_l1norm(tids)::FLOAT8 / {num_tranx}, tids, nextval('{assoc_loop_aux_sequence}') FROM(
+ SELECT DISTINCT ON({madlib_schema}.svec_to_string(set_list)) set_list, tids
FROM (
SELECT
{madlib_schema}.svec_minus(
@@ -401,9 +404,11 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
) t
WHERE {madlib_schema}.svec_l1norm(set_list)::INT = {iterp1} AND
{madlib_schema}.svec_l1norm(tids)::FLOAT8 >= {min_supp_tranx}
+ ) q
""".format(iterp1=iter + 1, **locals())
);
+ plpy.execute("""DROP SEQUENCE IF EXISTS {s};""".format(s=assoc_loop_aux_sequence))
plpy.execute("TRUNCATE TABLE assoc_rule_sets_loop");
plpy.execute("""
INSERT INTO assoc_rule_sets_loop(id, set_list, support, tids)
@@ -470,10 +475,14 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
begin_step_exec = time.time();
# generate all the final rules
+ assoc_rules_aux_tmp_sequence = unique_string(desp='s2')
+ plpy.execute("""CREATE SEQUENCE {s} START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1;
+ """.format(s=assoc_rules_aux_tmp_sequence))
plpy.execute("""
INSERT INTO assoc_rules_aux_tmp
- (pre, post, support, confidence, lift, conviction)
+ (ruleId, pre, post, support, confidence, lift, conviction)
SELECT
+ nextval('{assoc_rules_aux_tmp_sequence}'),
t.item[1],
t.item[2],
t.support_xy,
@@ -499,6 +508,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col,
(t.support_xy / x.support) >= {confidence}
""".format(**locals())
);
+ plpy.execute("""DROP SEQUENCE IF EXISTS {s};""".format(s=assoc_rules_aux_tmp_sequence))
# generate the readable rules
plpy.execute("DROP TABLE IF EXISTS pre_tmp_table");
diff --git a/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in b/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in
index a7f639b..4d8531a 100644
--- a/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in
+++ b/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in
@@ -529,7 +529,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gen_rules_from_cfp
INT
)
RETURNS SETOF TEXT[] AS 'MODULE_PATHNAME'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/bayes/bayes.py_in b/src/ports/postgres/modules/bayes/bayes.py_in
index b97929f..7507515 100644
--- a/src/ports/postgres/modules/bayes/bayes.py_in
+++ b/src/ports/postgres/modules/bayes/bayes.py_in
@@ -709,7 +709,6 @@ def create_prepared_data(**kwargs):
{attr_counts_sql}
m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (attr)');
ALTER TABLE {attrCountsSource} ADD PRIMARY KEY (attr);
- ANALYZE {attrCountsSource};
DROP TABLE IF EXISTS {attrValuesSource};
CREATE TEMPORARY TABLE {attrValuesSource}
@@ -718,7 +717,6 @@ def create_prepared_data(**kwargs):
m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (attr, value)');
ALTER TABLE {attrValuesSource} ADD PRIMARY KEY (attr, value);
- ANALYZE {attrValuesSource};
""".format(
attrCountsSource = kwargs['attrCountsSource'],
attrValuesSource = kwargs['attrValuesSource'],
@@ -741,7 +739,6 @@ def create_prepared_data(**kwargs):
if kwargs['whatToCreate'] == 'TABLE':
plpy.execute("""
ALTER TABLE {classPriorsDestName} ADD PRIMARY KEY (class);
- ANALYZE {classPriorsDestName};
""".format(**kwargs))
kwargs.update(dict(
@@ -786,7 +783,6 @@ def create_prepared_data(**kwargs):
if kwargs['whatToCreate'] == 'TABLE':
plpy.execute("""
ALTER TABLE {featureProbsDestName} ADD PRIMARY KEY (class, attr, value);
- ANALYZE {featureProbsDestName};
DROP TABLE {attrCountsSource};
DROP TABLE {attrValuesSource};
""".format(**kwargs))
@@ -795,7 +791,6 @@ def create_prepared_data(**kwargs):
if kwargs['whatToCreate'] == 'TABLE':
plpy.execute("""
ALTER TABLE {numericFeatureStatsDestName} ADD PRIMARY KEY (class, attr);
- ANALYZE {numericFeatureStatsDestName};
""".format(**kwargs))
@@ -1173,7 +1168,7 @@ def __verify_attr_num(sourceTable, attrColumn, numAttr):
Verify the number of attributes for each record is as expected.
"""
result = plpy.execute("""
- SELECT count(*) size FROM {0}
+ SELECT count(*) AS size FROM {0}
WHERE array_upper({1}, 1) <> {2}
""".format(sourceTable, attrColumn, numAttr)
)
diff --git a/src/ports/postgres/modules/convex/lmf.sql_in b/src/ports/postgres/modules/convex/lmf.sql_in
index e608d0a..700e34f 100644
--- a/src/ports/postgres/modules/convex/lmf.sql_in
+++ b/src/ports/postgres/modules/convex/lmf.sql_in
@@ -132,7 +132,7 @@ INSERT INTO lmf_data VALUES (999, 10000, 2.0);
-# Call the lmf_igd_run() stored procedure.
<pre class="example">
DROP TABLE IF EXISTS lmf_model;
-SELECT madlib.lmf_igd_run( 'lmf_model',
+SELECT MADLIB_SCHEMA.lmf_igd_run( 'lmf_model',
'lmf_data',
'row',
'col',
@@ -245,7 +245,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_transition(
scale_factor DOUBLE PRECISION)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_merge(
@@ -253,14 +253,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_merge(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -293,14 +293,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lmf_igd_distance(
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lmf_igd_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.lmf_result AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_lmf_igd_args(
@@ -309,7 +309,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_lmf_igd_args(
) RETURNS VOID
IMMUTABLE
CALLED ON NULL INPUT
-LANGUAGE c
+LANGUAGE c NOT FENCED
AS 'MODULE_PATHNAME', 'exec_sql_using'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -382,8 +382,9 @@ BEGIN
-- PostgreSQL 8.4:
-- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN
-- We therefore have to emulate.
+ DROP TABLE IF EXISTS pg_temp._madlib_lmf_igd_args;
+
PERFORM MADLIB_SCHEMA.internal_execute_using_lmf_igd_args($sql$
- DROP TABLE IF EXISTS pg_temp._madlib_lmf_igd_args;
CREATE TABLE pg_temp._madlib_lmf_igd_args AS
SELECT
$1 AS row_dim,
diff --git a/src/ports/postgres/modules/convex/mlp.sql_in b/src/ports/postgres/modules/convex/mlp.sql_in
index d6ce7ce..6caaae5 100644
--- a/src/ports/postgres/modules/convex/mlp.sql_in
+++ b/src/ports/postgres/modules/convex/mlp.sql_in
@@ -1442,7 +1442,7 @@ CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_transition(
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE;
+LANGUAGE C IMMUTABLE NOT FENCED;
CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_transition(
state DOUBLE PRECISION[],
@@ -1463,33 +1463,33 @@ CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_transition(
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE;
+LANGUAGE C IMMUTABLE NOT FENCED;
CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_merge(
state1 DOUBLE PRECISION[],
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_merge(
state1 DOUBLE PRECISION[],
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
/**
* @internal
@@ -1550,19 +1550,19 @@ CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_distance(
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.mlp_result AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_minibatch_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.mlp_result AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
-------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlp_classification(
@@ -1834,7 +1834,7 @@ CREATE FUNCTION MADLIB_SCHEMA.internal_predict_mlp(
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlp_classification(
diff --git a/src/ports/postgres/modules/convex/utils_regularization.sql_in b/src/ports/postgres/modules/convex/utils_regularization.sql_in
index 0bcef63..a4dc35f 100644
--- a/src/ports/postgres/modules/convex/utils_regularization.sql_in
+++ b/src/ports/postgres/modules/convex/utils_regularization.sql_in
@@ -12,7 +12,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_result (
in_state DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__utils_scales AS
'MODULE_PATHNAME', '__utils_var_scales_result'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -23,7 +23,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_transition (
dimension INTEGER
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'utils_var_scales_transition'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -33,7 +33,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_merge (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'utils_var_scales_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -42,14 +42,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_final (
state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'utils_var_scales_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_non_zero_std_final (
state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'utils_var_scales_non_zero_std_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -88,5 +88,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.utils_normalize_data (
std DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'utils_normalize_data'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/crf/crf.py_in b/src/ports/postgres/modules/crf/crf.py_in
index 2eaa1e8..e812134 100644
--- a/src/ports/postgres/modules/crf/crf.py_in
+++ b/src/ports/postgres/modules/crf/crf.py_in
@@ -209,7 +209,7 @@ def lincrf_train(schema_madlib, train_feature_tbl, train_featureset_tbl,
schema_madlib = schema_madlib))
# Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
- plpy.execute(""" ANALYZE {crf_weights_tbl} """.format(crf_weights_tbl = crf_weights_tbl))
+ # plpy.execute(""" ANALYZE {crf_weights_tbl} """.format(crf_weights_tbl = crf_weights_tbl))
return "CRF Train successful. Results stored in the specified CRF stats and weights table"
diff --git a/src/ports/postgres/modules/crf/crf.sql_in b/src/ports/postgres/modules/crf/crf.sql_in
index f9f6358..230df8e 100644
--- a/src/ports/postgres/modules/crf/crf.sql_in
+++ b/src/ports/postgres/modules/crf/crf.sql_in
@@ -708,33 +708,33 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lincrf_lbfgs_step_transition(
DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE;
+LANGUAGE C IMMUTABLE NOT FENCED;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lincrf_lbfgs_step_merge_states(
state1 DOUBLE PRECISION[],
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lincrf_lbfgs_step_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lincrf_lbfgs_converge(
/*+ state */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lincrf_lbfgs_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.lincrf_result AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
/**
* @internal
diff --git a/src/ports/postgres/modules/crf/crf_feature_gen.py_in b/src/ports/postgres/modules/crf/crf_feature_gen.py_in
index f55037f..7f9857b 100644
--- a/src/ports/postgres/modules/crf/crf_feature_gen.py_in
+++ b/src/ports/postgres/modules/crf/crf_feature_gen.py_in
@@ -138,7 +138,7 @@ def generate_train_features(schema_madlib, train_segment_tbl, regex_tbl, label_t
FROM """ + tmp1_feature + """;""")
# Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
- plpy.execute("""ANALYZE {tmp1_feature} """.format(tmp1_feature = tmp1_feature))
+ # plpy.execute("""ANALYZE {tmp1_feature} """.format(tmp1_feature = tmp1_feature))
plpy.execute("""DROP SEQUENCE IF EXISTS seq;
CREATE SEQUENCE seq START 1 INCREMENT 1;""")
@@ -152,7 +152,7 @@ def generate_train_features(schema_madlib, train_segment_tbl, regex_tbl, label_t
tmp_featureset = tmp_featureset))
# Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
- plpy.execute(""" ANALYZE {train_featureset_tbl} """.format(train_featureset_tbl = train_featureset_tbl))
+ # plpy.execute(""" ANALYZE {train_featureset_tbl} """.format(train_featureset_tbl = train_featureset_tbl))
rv = plpy.execute("""SELECT COUNT(*) AS total_feature FROM """ + train_featureset_tbl + """;""")
@@ -205,7 +205,7 @@ def generate_train_features(schema_madlib, train_segment_tbl, regex_tbl, label_t
dense_mtbl = dense_mtbl))
# Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
- plpy.execute(""" ANALYZE {train_feature_tbl} """.format(train_feature_tbl = train_feature_tbl))
+ # plpy.execute(""" ANALYZE {train_feature_tbl} """.format(train_feature_tbl = train_feature_tbl))
plpy.execute("""SET client_min_messages TO """ + str(origClientMinMessages[0]['setting']) + """;""")
@@ -412,8 +412,8 @@ m4_ifdef(`__HAS_ORDERED_AGGREGATES__', `
GROUP BY seg_text,label;""")
# Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
- plpy.execute(""" ANALYZE {viterbi_mtbl} """.format(viterbi_mtbl = viterbi_mtbl))
- plpy.execute(""" ANALYZE {viterbi_rtbl} """.format(viterbi_rtbl = viterbi_rtbl))
+ # plpy.execute(""" ANALYZE {viterbi_mtbl} """.format(viterbi_mtbl = viterbi_mtbl))
+ # plpy.execute(""" ANALYZE {viterbi_rtbl} """.format(viterbi_rtbl = viterbi_rtbl))
def _validate_label_tbl(label_tbl):
diff --git a/src/ports/postgres/modules/crf/viterbi.sql_in b/src/ports/postgres/modules/crf/viterbi.sql_in
index d3c20cd..8aa0b08 100644
--- a/src/ports/postgres/modules/crf/viterbi.sql_in
+++ b/src/ports/postgres/modules/crf/viterbi.sql_in
@@ -35,7 +35,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
* @returns the top1 label sequence, the last two elements in the array is used to calculate the top1 probability
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.vcrf_top1_label(mArray DOUBLE PRECISION[], rArray DOUBLE PRECISION[], nlabel int)
-returns int[] as 'MODULE_PATHNAME' language c strict;
+returns int[] as 'MODULE_PATHNAME' language c strict NOT FENCED;
/**
diff --git a/src/ports/postgres/modules/deep_learning/gpu_info_from_tf.py_in b/src/ports/postgres/modules/deep_learning/gpu_info_from_tf.py_in
deleted file mode 100644
index b1e59b1..0000000
--- a/src/ports/postgres/modules/deep_learning/gpu_info_from_tf.py_in
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-@file gpu_info_from_tf.py_in
-
-@brief This file prints out the gpu descriptions using tensorflow API. This file
-is intended to be called using subprocess. See madlib_keras_gpu_info.py_in
-for more details.
-"""
-
-from tensorflow.python.client import device_lib
-from keras import backend as K
-
-config = K.tf.ConfigProto()
-config.gpu_options.allow_growth = True
-sess = K.tf.Session(config=config)
-local_device_protos = device_lib.list_local_devices()
-K.clear_session()
-sess.close()
-if local_device_protos:
- for x in local_device_protos:
- if x.device_type == 'GPU':
- print x.physical_device_desc
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
deleted file mode 100644
index 605d439..0000000
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
+++ /dev/null
@@ -1,1036 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-"""
-@file input_data_preprocessor.py_in
-
-"""
-from math import ceil
-import plpy
-
-from internal.db_utils import get_distinct_col_levels
-from internal.db_utils import quote_literal
-from internal.db_utils import get_product_of_dimensions
-from utilities.minibatch_preprocessing import MiniBatchBufferSizeCalculator
-from utilities.control import OptimizerControl
-from utilities.control import HashaggControl
-from utilities.utilities import _assert
-from utilities.utilities import add_postfix
-from utilities.utilities import is_platform_pg
-from utilities.utilities import is_psql_char_type
-from utilities.utilities import is_valid_psql_type
-from utilities.utilities import is_var_valid
-from utilities.utilities import BOOLEAN, NUMERIC, ONLY_ARRAY, TEXT
-from utilities.utilities import py_list_to_sql_string
-from utilities.utilities import split_quoted_delimited_str
-from utilities.utilities import strip_end_quotes
-from utilities.utilities import unique_string
-from utilities.utilities import validate_module_input_params
-from utilities.utilities import get_seg_number
-from utilities.validate_args import input_tbl_valid
-from utilities.validate_args import get_expr_type
-
-from madlib_keras_helper import *
-import time
-
-NUM_CLASSES_COLNAME = "num_classes"
-
-class InputDataPreprocessorDL(object):
- def __init__(self, schema_madlib, source_table, output_table,
- dependent_varname, independent_varname, buffer_size,
- normalizing_const, num_classes, distribution_rules, module_name):
- self.schema_madlib = schema_madlib
- self.source_table = source_table
- self.output_table = output_table
- self.dependent_varname = dependent_varname
- self.independent_varname = independent_varname
- self.buffer_size = buffer_size
- self.normalizing_const = normalizing_const
- self.num_classes = num_classes
- self.distribution_rules = distribution_rules if distribution_rules else 'all_segments'
- self.module_name = module_name
- self.output_summary_table = None
- self.dependent_vartype = None
- self.independent_vartype = None
- self.gpu_config = '$__madlib__$all_segments$__madlib__$'
- if self.output_table:
- self.output_summary_table = add_postfix(self.output_table, "_summary")
-
- ## Validating input args prior to using them in _set_validate_vartypes()
- self._validate_args()
- self._set_validate_vartypes()
- self.dependent_levels = None
- # The number of padded zeros to include in 1-hot vector
- self.padding_size = 0
-
- def _set_one_hot_encoding_variables(self):
- """
- Set variables such as dependent_levels and padding_size.
- If necessary, NULLs are padded to dependent_levels list.
- """
- if self.dependent_levels:
- # if any class level was NULL in sql, that would show up as
- # None in self.dependent_levels. Replace all None with NULL
- # in the list.
- self.dependent_levels = ['NULL' if level is None else level
- for level in self.dependent_levels]
- self._validate_num_classes()
- # Try computing padding_size after running all necessary validations.
- if self.num_classes:
- self.padding_size = self.num_classes - len(self.dependent_levels)
-
- def _validate_num_classes(self):
- if self.num_classes is not None and \
- self.num_classes < len(self.dependent_levels):
- plpy.error("{0}: Invalid num_classes value specified. It must "\
- "be equal to or greater than distinct class values found "\
- "in table ({1}).".format(
- self.module_name, len(self.dependent_levels)))
-
- def _validate_distribution_table(self):
-
- input_tbl_valid(self.distribution_rules, self.module_name,
- error_suffix_str="""
- segments_to_use table ({self.distribution_rules}) doesn't exist.
- """.format(self=self))
- _assert(is_var_valid(self.distribution_rules, 'dbid'),
- "{self.module_name}: distribution rules table must contain dbib column".format(
- self=self))
- dbids = plpy.execute("""
- SELECT array_agg(dbid) AS dbids FROM gp_segment_configuration
- WHERE content >= 0 AND role = 'p'
- """)[0]['dbids']
- dist_result = plpy.execute("""
- SELECT array_agg(dbid) AS dbids,
- count(dbid) AS c1,
- count(DISTINCT dbid) AS c2
- FROM {0} """.format(self.distribution_rules))
-
- _assert(dist_result[0]['c1'] == dist_result[0]['c2'],
- '{self.module_name}: distribution rules table contains duplicate dbids'.format(
- self=self))
-
- for i in dist_result[0]['dbids']:
- _assert(i in dbids,
- '{self.module_name}: invalid dbid:{i} in the distribution rules table'.format(
- self=self, i=i))
-
- def get_one_hot_encoded_dep_var_expr(self):
- """
- :param dependent_varname: Name of the dependent variable
- :param num_classes: Number of class values to consider in 1-hot
- :return:
- This function returns a tuple of
- 1. A string with transformed dependent varname depending on it's type
- 2. All the distinct dependent class levels encoded as a string
-
- If dep_type == numeric[] , do not encode
- 1. dependent_varname = rings
- transformed_value = ARRAY[rings]
- 2. dependent_varname = ARRAY[a, b, c]
- transformed_value = ARRAY[a, b, c]
- else if dep_type in ("text", "boolean"), encode:
- 3. dependent_varname = rings (encoding)
- transformed_value = ARRAY[rings=1, rings=2, rings=3]
- """
- # Assuming the input NUMERIC[] is already one_hot_encoded,
- # so casting to INTEGER[]
- if is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY):
- return "{0}::{1}[]".format(self.dependent_varname, SMALLINT_SQL_TYPE)
-
- # For DL use case, we want to allow NULL as a valid class value,
- # so the query must have 'IS NOT DISTINCT FROM' instead of '='
- # like in the generic get_one_hot_encoded_expr() defined in
- # db_utils.py_in. We also have this optional 'num_classes' param
- # that affects the logic of 1-hot encoding. Since this is very
- # specific to input_preprocessor_dl for now, let's keep
- # it here instead of refactoring it out to a generic helper function.
- one_hot_encoded_expr = ["({0}) IS NOT DISTINCT FROM {1}".format(
- self.dependent_varname, c) for c in self.dependent_levels]
- if self.num_classes:
- one_hot_encoded_expr.extend(['false'
- for i in range(self.padding_size)])
- # In psql, we can't directly convert boolean to smallint, so we firstly
- # convert it to integer and then cast to smallint
- return 'ARRAY[{0}]::INTEGER[]::{1}[]'.format(
- ', '.join(one_hot_encoded_expr), SMALLINT_SQL_TYPE)
-
- def _get_independent_var_shape(self):
-
- shape = plpy.execute(
- "SELECT array_dims({0}) AS shape FROM {1} LIMIT 1".format(
- self.independent_varname, self.source_table))[0]['shape']
- return parse_shape(shape)
-
- def _get_dependent_var_shape(self):
-
- if self.num_classes:
- shape = [self.num_classes]
- elif self.dependent_levels:
- shape = [len(self.dependent_levels)]
- else:
- shape = plpy.execute(
- "SELECT array_dims({0}) AS shape FROM {1} LIMIT 1".format(
- self.dependent_varname, self.source_table))[0]['shape']
- shape = parse_shape(shape)
- return shape
-
- def input_preprocessor_dl(self, order_by_random=True):
- """
- Creates the output and summary table that does the following
- pre-processing operations on the input data:
- 1) Normalizes the independent variable.
- 2) Minibatches the normalized independent variable.
- 3) One-hot encodes the dependent variable.
- 4) Minibatches the one-hot encoded dependent variable.
- """
- # setup for 1-hot encoding
- self._set_one_hot_encoding_variables()
-
- # Generate random strings for TEMP tables
- series_tbl = unique_string(desp='series')
- dist_key_tbl = unique_string(desp='dist_key')
- normalized_tbl = unique_string(desp='normalized_table')
- batched_table = unique_string(desp='batched_table')
-
- # Used later in locals() for formatting queries
- x=MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
- y=MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
- float32=FLOAT32_SQL_TYPE
- dep_shape_col = add_postfix(y, "_shape")
- ind_shape_col = add_postfix(x, "_shape")
-
- ind_shape = self._get_independent_var_shape()
- ind_shape = ','.join([str(i) for i in ind_shape])
- dep_shape = self._get_dependent_var_shape()
- dep_shape = ','.join([str(i) for i in dep_shape])
-
- one_hot_dep_var_array_expr = self.get_one_hot_encoded_dep_var_expr()
-
- # skip normalization step if normalizing_const = 1.0
- if self.normalizing_const and (self.normalizing_const < 0.999999 or self.normalizing_const > 1.000001):
- rescale_independent_var = """{self.schema_madlib}.array_scalar_mult(
- {self.independent_varname}::{float32}[],
- (1/{self.normalizing_const})::{float32})
- """.format(**locals())
- else:
- self.normalizing_const = DEFAULT_NORMALIZING_CONST
- rescale_independent_var = "{self.independent_varname}::{float32}[]".format(**locals())
-
- # It's important that we shuffle all rows before batching for fit(), but
- # we can skip that for predict()
- order_by_clause = " ORDER BY RANDOM()" if order_by_random else ""
-
- # This query template will be used later in pg & gp specific code paths,
- # where {make_buffer_id} and {dist_by_buffer_id} are filled in
- batching_query = """
- CREATE TEMP TABLE {batched_table} AS SELECT
- {{make_buffer_id}} buffer_id,
- {self.schema_madlib}.agg_array_concat(
- ARRAY[x_norm::{float32}[]]) AS {x},
- {self.schema_madlib}.agg_array_concat(
- ARRAY[y]) AS {y},
- COUNT(*) AS count
- FROM {normalized_tbl}
- GROUP BY buffer_id
- {{dist_by_buffer_id}}
- """.format(**locals())
-
- # This query template will be used later in pg & gp specific code paths,
- # where {dist_key_col_comma} and {dist_by_dist_key} will be filled in
- bytea_query = """
- CREATE TABLE {self.output_table} AS SELECT
- {{dist_key_col_comma}}
- {self.schema_madlib}.array_to_bytea({x}) AS {x},
- {self.schema_madlib}.array_to_bytea({y}) AS {y},
- ARRAY[count,{ind_shape}]::SMALLINT[] AS {ind_shape_col},
- ARRAY[count,{dep_shape}]::SMALLINT[] AS {dep_shape_col},
- buffer_id
- FROM {batched_table}
- {{dist_by_dist_key}}
- """.format(**locals())
-
- if is_platform_pg():
- # used later for writing summary table
- self.distribution_rules = '$__madlib__$all_segments$__madlib__$'
-
- #
- # For postgres, we just need 3 simple queries:
- # 1-hot-encode/normalize + batching + bytea conversion
- #
-
- # see note in gpdb code branch (lower down) on
- # 1-hot-encoding of dependent var
- one_hot_sql = """
- CREATE TEMP TABLE {normalized_tbl} AS SELECT
- (ROW_NUMBER() OVER({order_by_clause}) - 1)::INTEGER as row_id,
- {rescale_independent_var} AS x_norm,
- {one_hot_dep_var_array_expr} AS y
- FROM {self.source_table}
- """.format(**locals())
-
- plpy.execute(one_hot_sql)
-
- self.buffer_size = self._get_buffer_size(1)
-
- # Used to format query templates with locals()
- make_buffer_id = 'row_id / {0} AS '.format(self.buffer_size)
- dist_by_dist_key = ''
- dist_by_buffer_id = ''
- dist_key_col_comma = ''
-
- # Disable hashagg since large number of arrays being concatenated
- # could result in excessive memory usage.
- with HashaggControl(False):
- # Batch rows with GROUP BY
- plpy.execute(batching_query.format(**locals()))
-
- plpy.execute("DROP TABLE {0}".format(normalized_tbl))
-
- # Convert to BYTEA and output final (permanent table)
- plpy.execute(bytea_query.format(**locals()))
-
- plpy.execute("DROP TABLE {0}".format(batched_table))
-
- self._create_output_summary_table()
-
- return
-
- # Done with postgres, rest is all for gpdb
- #
- # This gpdb code path is far more complex, and depends on
- # how the user wishes to distribute the data. Even if
- # it's to be spread evenly across all segments, we still
- # need to do some extra work to ensure that happens.
-
- if self.distribution_rules == 'all_segments':
- all_segments = True
- self.distribution_rules = '$__madlib__$all_segments$__madlib__$'
- num_segments = get_seg_number()
- else:
- all_segments = False
-
- if self.distribution_rules == 'gpu_segments':
- #TODO can we reuse the function `get_accessible_gpus_for_seg` from
- # madlib_keras_helper
- gpu_info_table = unique_string(desp='gpu_info')
- plpy.execute("""
- SELECT {self.schema_madlib}.gpu_configuration('{gpu_info_table}')
- """.format(**locals()))
- gpu_query = """
- SELECT array_agg(DISTINCT(hostname)) as gpu_config
- FROM {gpu_info_table}
- """.format(**locals())
- gpu_query_result = plpy.execute(gpu_query)[0]['gpu_config']
- if not gpu_query_result:
- plpy.error("{self.module_name}: No GPUs configured on hosts.".format(self=self))
- plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
-
- gpu_config_hostnames = "ARRAY{0}".format(gpu_query_result)
- # find hosts with gpus
- get_segment_query = """
- SELECT array_agg(content) as segment_ids,
- array_agg(dbid) as dbid,
- count(*) as count
- FROM gp_segment_configuration
- WHERE content != -1 AND role = 'p'
- AND hostname=ANY({gpu_config_hostnames})
- """.format(**locals())
- segment_ids_result = plpy.execute(get_segment_query)[0]
-
- self.gpu_config = "ARRAY{0}".format(sorted(segment_ids_result['segment_ids']))
- self.distribution_rules = "ARRAY{0}".format(sorted(segment_ids_result['dbid']))
-
- num_segments = segment_ids_result['count']
-
- elif not all_segments: # Read from a table with dbids to distribute the data
- self._validate_distribution_table()
- gpu_query = """
- SELECT array_agg(content) as gpu_config,
- array_agg(gp_segment_configuration.dbid) as dbid
- FROM {self.distribution_rules} JOIN gp_segment_configuration
- ON {self.distribution_rules}.dbid = gp_segment_configuration.dbid
- """.format(**locals())
- gpu_query_result = plpy.execute(gpu_query)[0]
- self.gpu_config = "ARRAY{0}".format(sorted(gpu_query_result['gpu_config']))
- num_segments = plpy.execute("SELECT count(*) as count FROM {self.distribution_rules}".format(**locals()))[0]['count']
- self.distribution_rules = "ARRAY{0}".format(sorted(gpu_query_result['dbid']))
-
- join_key = 't.buffer_id % {num_segments}'.format(**locals())
-
- if not all_segments:
- join_key = '({self.gpu_config})[{join_key} + 1]'.format(**locals())
-
- # Create large temp table such that there is atleast 1 row on each segment
- # Using 999999 would distribute data(atleast 1 row on each segment) for
- # a cluster as large as 20000
- dist_key_col = DISTRIBUTION_KEY_COLNAME
- query = """
- CREATE TEMP TABLE {series_tbl} AS
- SELECT generate_series(0, 999999) {dist_key_col}
- DISTRIBUTED BY ({dist_key_col})
- """.format(**locals())
-
- plpy.execute(query)
-
- # Used in locals() to format queries, including template queries
- # bytea_query & batching_query defined in section common to
- # pg & gp (very beginning of this function)
- dist_by_dist_key = 'DISTRIBUTED BY ({dist_key_col})'.format(**locals())
- dist_by_buffer_id = 'DISTRIBUTED BY (buffer_id)'
- dist_key_col_comma = dist_key_col + ' ,'
- make_buffer_id = ''
-
- dist_key_query = """
- CREATE TEMP TABLE {dist_key_tbl} AS
- SELECT min({dist_key_col}) AS {dist_key_col}
- FROM {series_tbl}
- GROUP BY gp_segment_id
- DISTRIBUTED BY ({dist_key_col})
- """.format(**locals())
-
- plpy.execute(dist_key_query)
-
- plpy.execute("DROP TABLE {0}".format(series_tbl))
-
- # Always one-hot encode the dependent var. For now, we are assuming
- # that input_preprocessor_dl will be used only for deep
- # learning and mostly for classification. So make a strong
- # assumption that it is only for classification, so one-hot
- # encode the dep var, unless it's already a numeric array in
- # which case we assume it's already one-hot encoded.
-
- # While 1-hot-encoding is done, we also normalize the independent
- # var and randomly shuffle the rows on each segment. (The dist key
- # we're adding avoids any rows moving between segments. This may
- # make things slightly less random, but helps with speed--probably
- # a safe tradeoff to make.)
-
- norm_tbl = unique_string(desp='norm_table')
-
- one_hot_sql = """
- CREATE TEMP TABLE {norm_tbl} AS
- SELECT {dist_key_col},
- {rescale_independent_var} AS x_norm,
- {one_hot_dep_var_array_expr} AS y
- FROM {self.source_table} s JOIN {dist_key_tbl} AS d
- ON (s.gp_segment_id = d.gp_segment_id)
- {order_by_clause}
- DISTRIBUTED BY ({dist_key_col})
- """.format(**locals())
- plpy.execute(one_hot_sql)
-
- rows_per_seg_tbl = unique_string(desp='rows_per_seg')
- start_rows_tbl = unique_string(desp='start_rows')
-
- # Generate rows_per_segment table; this small table will
- # just have one row on each segment containing the number
- # of rows on that segment in the norm_tbl
- sql = """
- CREATE TEMP TABLE {rows_per_seg_tbl} AS SELECT
- COUNT(*) as rows_per_seg,
- {dist_key_col}
- FROM {norm_tbl}
- GROUP BY {dist_key_col}
- DISTRIBUTED BY ({dist_key_col})
- """.format(**locals())
-
- plpy.execute(sql)
-
- # Generate start_rows_tbl from rows_per_segment table.
- # This assigns a start_row number for each segment based on
- # the sum of all rows in previous segments. These will be
- # added to the row numbers within each segment to get an
- # absolute index into the table. All of this is to accomplish
- # the equivalent of ROW_NUMBER() OVER() on the whole table,
- # but this way is much faster because we don't have to do an
- # N:1 Gather Motion (moving entire table to a single segment
- # and scanning through it).
- #
- sql = """
- CREATE TEMP TABLE {start_rows_tbl} AS SELECT
- {dist_key_col},
- SUM(rows_per_seg) OVER (ORDER BY gp_segment_id) - rows_per_seg AS start_row
- FROM {rows_per_seg_tbl}
- DISTRIBUTED BY ({dist_key_col})
- """.format(**locals())
-
- plpy.execute(sql)
-
- plpy.execute("DROP TABLE {0}".format(rows_per_seg_tbl))
-
- self.buffer_size = self._get_buffer_size(num_segments)
-
- # The query below assigns slot_id's to each row within
- # a segment, computes a row_id by adding start_row for
- # that segment to it, then divides by buffer_size to make
- # this into a buffer_id
- # ie:
- # buffer_id = row_id / buffer_size
- # row_id = start_row + slot_id
- # slot_id = ROW_NUMBER() OVER(PARTITION BY <dist key>)::INTEGER
- #
- # Instead of partitioning by gp_segment_id itself, we
- # use __dist_key__ col instead. This is the same partition,
- # since there's a 1-to-1 mapping between the columns; but
- # using __dist_key__ avoids an extra Redistribute Motion.
- #
- # Note: even though the ordering of these two columns is
- # different, this doesn't matter as each segment is being
- # numbered separately (only the start_row is different,
- # and those are fixed to the correct segments by the JOIN
- # condition.
-
- sql = """
- CREATE TEMP TABLE {normalized_tbl} AS SELECT
- {dist_key_col},
- x_norm,
- y,
- (ROW_NUMBER() OVER( PARTITION BY {dist_key_col} ))::INTEGER as slot_id,
- ((start_row +
- (ROW_NUMBER() OVER( PARTITION BY {dist_key_col} ) - 1)
- )::INTEGER / {self.buffer_size}
- ) AS buffer_id
- FROM {norm_tbl} JOIN {start_rows_tbl}
- USING ({dist_key_col})
- ORDER BY buffer_id
- DISTRIBUTED BY (slot_id)
- """.format(**locals())
-
- plpy.execute(sql) # label buffer_id's
-
- # A note on DISTRIBUTED BY (slot_id) in above query:
- #
- # In the next query, we'll be doing the actual batching. Due
- # to the GROUP BY, gpdb will Redistribute on buffer_id. We could
- # avoid this by using DISTRIBUTED BY (buffer_id) in the above
- # (buffer-labelling) query. But this also causes the GROUP BY
- # to use single-stage GroupAgg instead of multistage GroupAgg,
- # which for unknown reasons is *much* slower and often runs out
- # of VMEM unless it's set very high!
-
- plpy.execute("DROP TABLE {norm_tbl}, {start_rows_tbl}".format(**locals()))
-
- # Disable optimizer (ORCA) for platforms that use it
- # since we want to use a groupagg instead of hashagg
- with OptimizerControl(False):
- with HashaggControl(False):
- # Run actual batching query
- plpy.execute(batching_query.format(**locals()))
-
- plpy.execute("DROP TABLE {0}".format(normalized_tbl))
-
- if not all_segments: # remove any segments we don't plan to use
- sql = """
- DELETE FROM {dist_key_tbl}
- WHERE NOT gp_segment_id = ANY({self.gpu_config})
- """.format(**locals())
-
- plpy.execute("ANALYZE {dist_key_tbl}".format(**locals()))
- plpy.execute("ANALYZE {batched_table}".format(**locals()))
-
- # Redistribute from buffer_id to dist_key
- #
- # This has to be separate from the batching query, because
- # we found that adding DISTRIBUTED BY (dist_key) to that
- # query causes it to run out of VMEM on large datasets such
- # as places100. Possibly this is because the memory available
- # for GroupAgg has to be shared with an extra slice if they
- # are part of the same query.
- #
- # We also tried adding this to the BYTEA conversion query, but
- # that resulted in slower performance than just keeping it
- # separate.
- #
- sql = """CREATE TEMP TABLE {batched_table}_dist_key AS
- SELECT {dist_key_col}, t.*
- FROM {batched_table} t
- JOIN {dist_key_tbl} d
- ON {join_key} = d.gp_segment_id
- DISTRIBUTED BY ({dist_key_col})
- """.format(**locals())
-
- # match buffer_id's with dist_keys
- plpy.execute(sql)
-
- sql = """DROP TABLE {batched_table}, {dist_key_tbl};
- ALTER TABLE {batched_table}_dist_key RENAME TO {batched_table}
- """.format(**locals())
- plpy.execute(sql)
-
- # Convert batched table to BYTEA and output as final (permanent) table
- plpy.execute(bytea_query.format(**locals()))
-
- plpy.execute("DROP TABLE {0}".format(batched_table))
-
- # Create summary table
- self._create_output_summary_table()
-
- def _create_output_summary_table(self):
- class_level_str='NULL::{0}[]'.format(self.dependent_vartype)
- if self.dependent_levels:
- # Update dependent_levels to include NULL when
- # num_classes > len(self.dependent_levels)
- if self.num_classes:
- self.dependent_levels.extend(['NULL'
- for i in range(self.padding_size)])
- else:
- self.num_classes = len(self.dependent_levels)
- class_level_str=py_list_to_sql_string(
- self.dependent_levels, array_type=self.dependent_vartype,
- long_format=True)
- if self.num_classes is None:
- self.num_classes = 'NULL::INTEGER'
- query = """
- CREATE TABLE {self.output_summary_table} AS
- SELECT
- $__madlib__${self.source_table}$__madlib__$::TEXT AS source_table,
- $__madlib__${self.output_table}$__madlib__$::TEXT AS output_table,
- $__madlib__${self.dependent_varname}$__madlib__$::TEXT AS {dependent_varname_colname},
- $__madlib__${self.independent_varname}$__madlib__$::TEXT AS {independent_varname_colname},
- $__madlib__${self.dependent_vartype}$__madlib__$::TEXT AS {dependent_vartype_colname},
- {class_level_str} AS {class_values_colname},
- {self.buffer_size} AS buffer_size,
- {self.normalizing_const}::{FLOAT32_SQL_TYPE} AS {normalizing_const_colname},
- {self.num_classes} AS {num_classes_colname},
- {self.distribution_rules} AS distribution_rules,
- {self.gpu_config} AS {internal_gpu_config}
- """.format(self=self, class_level_str=class_level_str,
- dependent_varname_colname=DEPENDENT_VARNAME_COLNAME,
- independent_varname_colname=INDEPENDENT_VARNAME_COLNAME,
- dependent_vartype_colname=DEPENDENT_VARTYPE_COLNAME,
- class_values_colname=CLASS_VALUES_COLNAME,
- normalizing_const_colname=NORMALIZING_CONST_COLNAME,
- num_classes_colname=NUM_CLASSES_COLNAME,
- internal_gpu_config=INTERNAL_GPU_CONFIG,
- FLOAT32_SQL_TYPE=FLOAT32_SQL_TYPE)
- plpy.execute(query)
-
- def _validate_args(self):
- validate_module_input_params(
- self.source_table, self.output_table, self.independent_varname,
- self.dependent_varname, self.module_name, None,
- [self.output_summary_table])
- if self.buffer_size is not None:
- _assert(self.buffer_size > 0,
- "{0}: The buffer size has to be a "
- "positive integer or NULL.".format(self.module_name))
- if self.normalizing_const is not None:
- _assert(self.normalizing_const > 0,
- "{0}: The normalizing constant has to be a "
- "positive integer or NULL.".format(self.module_name))
-
- def _set_validate_vartypes(self):
- self.independent_vartype = get_expr_type(self.independent_varname,
- self.source_table)
- self.dependent_vartype = get_expr_type(self.dependent_varname,
- self.source_table)
- num_of_independent_cols = split_quoted_delimited_str(self.independent_varname)
- _assert(len(num_of_independent_cols) == 1,
- "Invalid independent_varname: only one column name is allowed "
- "as input.")
- _assert(is_valid_psql_type(self.independent_vartype,
- NUMERIC | ONLY_ARRAY),
- "Invalid independent variable type, should be an array of "
- "one of {0}".format(','.join(NUMERIC)))
- # The dependent variable needs to be either:
- # 1. NUMERIC, TEXT OR BOOLEAN, which we always one-hot encode
- # 2. NUMERIC ARRAY, which we assume it is already one-hot encoded, and we
- # just cast it the INTEGER ARRAY
- num_of_dependent_cols = split_quoted_delimited_str(self.dependent_varname)
- _assert(len(num_of_dependent_cols) == 1,
- "Invalid dependent_varname: only one column name is allowed "
- "as input.")
- _assert((is_valid_psql_type(self.dependent_vartype, NUMERIC | TEXT | BOOLEAN) or
- is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY)),
- """Invalid dependent variable type, should be one of the types in this list:
- numeric, text, boolean, or numeric array""")
-
- def get_distinct_dependent_levels(self, table, dependent_varname,
- dependent_vartype):
- # Refactoring this out into the parent class to ensure include_nulls
- # is passed in as true for both training and validation tables
- return get_distinct_col_levels(table, dependent_varname,
- dependent_vartype, include_nulls=True)
-
- def _get_buffer_size(self, num_segments):
- num_rows_in_tbl = plpy.execute("""
- SELECT count(*) AS cnt FROM {0}
- """.format(self.source_table))[0]['cnt']
- buffer_size_calculator = MiniBatchBufferSizeCalculator()
- indepdent_var_dim = get_product_of_dimensions(self.source_table,
- self.independent_varname)
- buffer_size = buffer_size_calculator.calculate_default_buffer_size(
- self.buffer_size, num_rows_in_tbl, indepdent_var_dim, num_segments)
- num_buffers = num_segments * ceil((1.0 * num_rows_in_tbl) / buffer_size / num_segments)
- return int(ceil(num_rows_in_tbl / num_buffers))
-
-class ValidationDataPreprocessorDL(InputDataPreprocessorDL):
- def __init__(self, schema_madlib, source_table, output_table,
- dependent_varname, independent_varname,
- training_preprocessor_table, buffer_size, distribution_rules,
- **kwargs):
- """
- This prepares the variables that are required by
- InputDataPreprocessorDL.
- """
- self.module_name = "validation_preprocessor_dl"
- self.training_preprocessor_table = training_preprocessor_table
- summary_table = self._validate_and_process_training_preprocessor_table()
- num_classes = summary_table[NUM_CLASSES_COLNAME]
- InputDataPreprocessorDL.__init__(
- self, schema_madlib, source_table, output_table,
- dependent_varname, independent_varname, buffer_size,
- summary_table[NORMALIZING_CONST_COLNAME], num_classes,
- distribution_rules, self.module_name)
- # Update value of dependent_levels from training batch summary table.
- self.dependent_levels = self._get_dependent_levels(
- summary_table[CLASS_VALUES_COLNAME],
- summary_table[DEPENDENT_VARTYPE_COLNAME])
-
- def _get_dependent_levels(self, training_dependent_levels,
- training_dependent_vartype):
- """
- Return the distinct dependent levels to be considered for
- one-hot encoding the dependent var. This is inferred from
- the class_values column in the training_preprocessor_table
- summary table. Note that class_values in that summary table
- already has padding in it, so we have to strip it out here
- in that case.
- This function also quotes class levels if they are text.
- """
- # Validate that dep var type is exactly the same as what was in
- # trainig_preprocessor_table's input.
- _assert(self.dependent_vartype == training_dependent_vartype,
- "{0}: the dependent variable's type in {1} must be {2}.".format(
- self.module_name, self.source_table,
- training_dependent_vartype))
- # training_dependent_levels is the class_values column from the
- # training batch summary table. This already has the padding with
- # NULLs in it based on num_classes that was provided to
- # training_preprocessor_dl(). We have to work our way backwards
- # to strip out those trailing NULLs from class_values, since
- # they will anyway get added later in
- # InputDataPreprocessorDL._set_one_hot_encoding_variables.
- dependent_levels = strip_trailing_nulls_from_class_values(
- training_dependent_levels)
- if training_dependent_levels:
- dependent_levels_val_data = self.get_distinct_dependent_levels(
- self.source_table, self.dependent_varname,
- self.dependent_vartype)
- unquoted_dependent_levels_val_data = [strip_end_quotes(level, "'")
- for level in dependent_levels_val_data]
- # Assert to check if the class values in validation data is a subset
- # of the class values in training data.
- _assert(set(unquoted_dependent_levels_val_data).issubset(set(dependent_levels)),
- "{0}: the class values in {1} ({2}) should be a "
- "subset of class values in {3} ({4})".format(
- self.module_name, self.source_table,
- unquoted_dependent_levels_val_data,
- self.training_preprocessor_table, dependent_levels))
- if is_psql_char_type(self.dependent_vartype):
- dependent_levels = [quote_literal(level) if level is not None else level
- for level in dependent_levels]
- return dependent_levels
-
- def _validate_and_process_training_preprocessor_table(self):
- """
- Validate training_preprocessor_table param passed. That and
- the corresponding summary tables must exist. The summary
- table must also have columns such as normalizing_const,
- class_values, num_classes and dependent_vartype in it.
- """
- input_tbl_valid(self.training_preprocessor_table, self.module_name)
- training_summary_table = add_postfix(
- self.training_preprocessor_table, "_summary")
- input_tbl_valid(training_summary_table, self.module_name,
- error_suffix_str="Please ensure that table '{0}' "
- "has been preprocessed using "
- "training_preprocessor_dl()."
- .format(self.training_preprocessor_table))
- summary_table = plpy.execute("SELECT * FROM {0} LIMIT 1".format(
- training_summary_table))[0]
- _assert(NORMALIZING_CONST_COLNAME in summary_table,
- "{0}: Expected column {1} in {2}.".format(
- self.module_name, NORMALIZING_CONST_COLNAME,
- training_summary_table))
- _assert(CLASS_VALUES_COLNAME in summary_table,
- "{0}: Expected column {1} in {2}.".format(
- self.module_name, CLASS_VALUES_COLNAME,
- training_summary_table))
- _assert(NUM_CLASSES_COLNAME in summary_table,
- "{0}: Expected column {1} in {2}.".format(
- self.module_name, NUM_CLASSES_COLNAME,
- training_summary_table))
- _assert(DEPENDENT_VARTYPE_COLNAME in summary_table,
- "{0}: Expected column {1} in {2}.".format(
- self.module_name, DEPENDENT_VARTYPE_COLNAME,
- training_summary_table))
- return summary_table
-
- def validation_preprocessor_dl(self):
- self.input_preprocessor_dl(order_by_random=False)
-
-class TrainingDataPreprocessorDL(InputDataPreprocessorDL):
- def __init__(self, schema_madlib, source_table, output_table,
- dependent_varname, independent_varname, buffer_size,
- normalizing_const, num_classes, distribution_rules,
- **kwargs):
- """
- This prepares the variables that are required by
- InputDataPreprocessorDL.
- """
- self.module_name = "training_preprocessor_dl"
- InputDataPreprocessorDL.__init__(
- self, schema_madlib, source_table, output_table,
- dependent_varname, independent_varname, buffer_size,
- normalizing_const, num_classes, distribution_rules,
- self.module_name)
- # Update default value of dependent_levels in superclass
- self.dependent_levels = self._get_dependent_levels()
-
- def _get_dependent_levels(self):
- """
- Return the distinct dependent levels to be considered for
- one-hot encoding the dependent var. class level values of
- type text are quoted.
- """
- if is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY):
- dependent_levels = None
- else:
- dependent_levels = get_distinct_col_levels(
- self.source_table, self.dependent_varname,
- self.dependent_vartype, include_nulls=True)
- return dependent_levels
-
- def training_preprocessor_dl(self):
- self.input_preprocessor_dl(order_by_random=True)
-
-class InputDataPreprocessorDocumentation:
- @staticmethod
- def validation_preprocessor_dl_help(schema_madlib, message):
- method = "validation_preprocessor_dl"
- summary = """
- ----------------------------------------------------------------
- SUMMARY
- ----------------------------------------------------------------
- For Deep Learning based techniques such as Convolutional Neural Nets,
- the input data is mostly images. These images can be represented as an
- array of numbers where each element represents a pixel/color intensity.
- It is standard practice to normalize the image data before use.
- minibatch_preprocessor() is for general use-cases, but for deep learning
- based use-cases we provide training_preprocessor_dl() that is
- light-weight and is specific to image datasets.
-
- If you want to evaluate the model, a validation dataset has to
- be prepared. This validation data has to be in the same format
- as the corresponding batched training data used for training, i.e.,
- the two datasets must be normalized using the same normalizing
- constant, and the one-hot encoding of the dependent variable must
- follow the same convention. validation_preprocessor_dl() can be
- used to pre-process the validation data. To ensure that the format
- is similar to the corresponding training data, this function takes
- the output table name of training_preprocessor_dl() as an input
- param.
-
- For more details on function usage:
- SELECT {schema_madlib}.{method}('usage')
- """.format(**locals())
-
- usage = """
- ---------------------------------------------------------------------------
- USAGE
- ---------------------------------------------------------------------------
- SELECT {schema_madlib}.{method}(
- source_table, -- TEXT. Name of the table containing input
- data. Can also be a view.
- output_table, -- TEXT. Name of the output table for
- mini-batching.
- dependent_varname, -- TEXT. Name of the dependent variable column.
- independent_varname, -- TEXT. Name of the independent variable
- column.
- training_preprocessor_table, -- TEXT. packed training data table.
- buffer_size -- INTEGER. Default computed automatically.
- Number of source input rows to pack into a buffer.
- distribution_rules -- TEXT. Default: 'all_segments'. Specifies how to
- distribute the 'output_table'. This is important
- for how the fit function will use resources on the
- cluster. The default 'all_segments' means the
- 'output_table' will be distributed to all segments
- in the database cluster.
- );
-
-
- ---------------------------------------------------------------------------
- OUTPUT
- ---------------------------------------------------------------------------
- The output table produced by validation_preprocessor_dl contains the
- following columns:
-
- buffer_id -- INTEGER. Unique id for packed table.
- dependent_varname -- BYTEA. Packed array of dependent variables.
- independent_varname -- BYTEA. Packed array of independent
- variables.
- dependent_varname -- TEXT. Shape of the dependent variable buffer.
- independent_varname -- TEXT. Shape of the independent variable buffer.
-
- ---------------------------------------------------------------------------
- The algorithm also creates a summary table named <output_table>_summary
- that has the following columns:
-
- source_table -- Source table name.
- output_table -- Output table name from preprocessor.
- dependent_varname -- Dependent variable values from the original table
- (encoded by one_hot_encode, if specified).
- independent_varname -- Independent variable values from the original
- table.
- dependent_vartype -- Type of the dependent variable from the
- original table.
- class_values -- Class values of the dependent variable
- (‘NULL’(as TEXT type) for non
- categorical vars).
- buffer_size -- Buffer size used in preprocessing step.
- normalizing_const -- Normalizing constant used for standardizing.
- arrays in independent_varname.
- num_classes -- num_classes value passed by user while
- generating training_preprocessor_table.
- gpu_config -- List of segment id's the data is distributed
- on depending on the 'distribution_rules' parameter
- specified as input. Set to 'all_segments' if
- 'distribution_rules' is specified as 'all_segments'.
-
- ---------------------------------------------------------------------------
- """.format(**locals())
-
- if not message:
- return summary
- elif message.lower() in ('usage', 'help', '?'):
- return usage
- return """
- No such option. Use "SELECT {schema_madlib}.{method}()"
- for help.
- """.format(**locals())
-
- @staticmethod
- def training_preprocessor_dl_help(schema_madlib, message):
- method = "training_preprocessor_dl"
- summary = """
- ----------------------------------------------------------------
- SUMMARY
- ----------------------------------------------------------------
- For Deep Learning based techniques such as Convolutional Neural Nets,
- the input data is mostly images. These images can be represented as an
- array of numbers where each element represents a pixel/color intensity.
- It is standard practice to normalize the image data before use.
- minibatch_preprocessor() is for general use-cases, but for deep learning
- based use-cases we provide training_preprocessor_dl() that is
- light-weight and is specific to image datasets.
-
- The normalizing constant is parameterized, and can be specified based
- on the kind of image data used.
-
- An optional param named num_classes can be used to specify the length
- of the one-hot encoded array for the dependent variable. This value if
- specified must be greater than equal to the total number of distinct
- class values found in the input table.
-
- For more details on function usage:
- SELECT {schema_madlib}.{method}('usage')
- """.format(**locals())
-
- usage = """
- ---------------------------------------------------------------------------
- USAGE
- ---------------------------------------------------------------------------
- SELECT {schema_madlib}.{method}(
- source_table, -- TEXT. Name of the table containing input
- data. Can also be a view.
- output_table, -- TEXT. Name of the output table for
- mini-batching.
- dependent_varname, -- TEXT. Name of the dependent variable column.
- independent_varname, -- TEXT. Name of the independent variable
- column.
- buffer_size -- INTEGER. Default computed automatically.
- Number of source input rows to pack into a buffer.
- normalizing_const -- REAL. Default 1.0. The normalizing constant to
- use for standardizing arrays in independent_varname.
- num_classes -- INTEGER. Default NULL. Number of class labels
- to be considered for 1-hot encoding. If NULL,
- the 1-hot encoded array length will be equal to
- the number of distinct class values found in the
- input table.
- distribution_rules -- TEXT. Default: 'all_segments'. Specifies how to
- distribute the 'output_table'. This is important
- for how the fit function will use resources on the
- cluster. The default 'all_segments' means the
- 'output_table' will be distributed to all segments
- in the database cluster.
- );
-
-
- ---------------------------------------------------------------------------
- OUTPUT
- ---------------------------------------------------------------------------
- The output table produced by MiniBatch Preprocessor contains the
- following columns:
-
- buffer_id -- INTEGER. Unique id for packed table.
- dependent_varname -- BYTEA. Packed array of dependent variables.
- independent_varname -- BYTEA. Packed array of independent
- variables.
- dependent_varname -- TEXT. Shape of the dependent variable buffer.
- independent_varname -- TEXT. Shape of the independent variable buffer.
-
- ---------------------------------------------------------------------------
- The algorithm also creates a summary table named <output_table>_summary
- that has the following columns:
-
- source_table -- Source table name.
- output_table -- Output table name from preprocessor.
- dependent_varname -- Dependent variable values from the original table
- (encoded by one_hot_encode, if specified).
- independent_varname -- Independent variable values from the original
- table.
- dependent_vartype -- Type of the dependent variable from the
- original table.
- class_values -- Class values of the dependent variable
- (‘NULL’(as TEXT type) for non
- categorical vars).
- buffer_size -- Buffer size used in preprocessing step.
- normalizing_const -- Normalizing constant used for standardizing
- arrays in independent_varname.
- num_classes -- num_classes input param passed to function.
- gpu_config -- List of segment id's the data is distributed
- on depending on the 'distribution_rules' param
- specified as input. Set to 'all_segments' if
- 'distribution_rules' is specified as 'all_segments'.
-
-
- ---------------------------------------------------------------------------
- """.format(**locals())
-
- if not message:
- return summary
- elif message.lower() in ('usage', 'help', '?'):
- return usage
- return """
- No such option. Use "SELECT {schema_madlib}.{method}()"
- for help.
- """.format(**locals())
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in
deleted file mode 100644
index f243417..0000000
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in
+++ /dev/null
@@ -1,1056 +0,0 @@
-/* ----------------------------------------------------------------------- */
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * @file input_preprocessor_dl.sql_in
- * @brief Utilities to prepare input image data for use by deep learning modules.
- * @date December 2018
- *
- */
-/* ----------------------------------------------------------------------- */
-
-m4_include(`SQLCommon.m4')
-
-/**
-@addtogroup grp_input_preprocessor_dl
-
-@brief Utilities that prepare input image data for use by deep learning
-modules.
-
-\warning <em> This MADlib method is still in early stage development.
-Interface and implementation are subject to change. </em>
-
-<div class="toc"><b>Contents</b><ul>
-<li class="level1"><a href="#training_preprocessor_dl">Preprocessor for Training Image Data</a></li>
-<li class="level1"><a href="#validation_preprocessor_dl">Preprocessor for Validation Image Data</a></li>
-<li class="level1"><a href="#output">Output Tables</a></li>
-<li class="level1"><a href="#example">Examples</a></li>
-<li class="level1"><a href="#references">References</a></li>
-<li class="level1"><a href="#related">Related Topics</a></li>
-</ul></div>
-
-This preprocessor is a utility that prepares image data for use
-by frameworks like Keras and TensorFlow that support mini-batching
-as an optimization option. The advantage of using mini-batching is that
-it can perform better than stochastic gradient descent
-because it uses more than one training example at a time, typically
-resulting in faster and smoother convergence [1].
-
-Images can be
-represented as an array of numbers
-where each element represents grayscale,
-RGB or other channel values for each
-pixel in the image. It is standard practice to
-normalize the image data before training.
-The normalizing constant in this module is parameterized,
-so it can be set depending on
-the format of image data used.
-
-There are two versions of the preprocessor:
-training_preprocessor_dl() preprocesses input image data to be
-used for training a deep learning model, while
-validation_preprocessor_dl() preprocesses validation
-image data used for model evaluation.
-
-@anchor training_preprocessor_dl
-@par Preprocessor for Training Image Data
-
-<pre class="syntax">
-training_preprocessor_dl(source_table,
- output_table,
- dependent_varname,
- independent_varname,
- buffer_size,
- normalizing_const,
- num_classes,
- distribution_rules
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
- <dt>source_table</dt>
- <dd>TEXT. Name of the table containing training dataset.
- Can also be a view.
- </dd>
-
- <dt>output_table</dt>
- <dd>TEXT. Name of the output table from the training preprocessor which
- will be used as input to algorithms that support mini-batching.
- Note that the arrays packed into the output table are shuffled
- and normalized, by dividing each element in the independent variable array
- by the optional 'normalizing_const' parameter. For performance reasons,
- packed arrays are converted to PostgreSQL bytea format, which is a
- variable-length binary string.
-
- In the case a validation data set is used (see
- later on this page), this output table is also used
- as an input to the validation preprocessor
- so that the validation and training image data are
- both preprocessed in an identical manner.
- </dd>
-
- <dt>dependent_varname</dt>
- <dd>TEXT. Name of the dependent variable column.
- @note The mini-batch preprocessor automatically 1-hot encodes
- dependent variables of all types. The exception is numeric array types
- (integer and float), where we assume these are already 1-hot encoded,
- so these will just be passed through as is.
- </dd>
-
- <dt>independent_varname</dt>
- <dd>TEXT. Name of the independent variable column. The column must be
- a numeric array type.
- </dd>
-
- <dt>buffer_size (optional)</dt>
- <dd>INTEGER, default: computed. Buffer size is the number of rows from
- the source table that are packed into one row of the preprocessor output
- table. In the case of images, the source table will have one image per row,
- and the output table will have multiple images per row. The default value is
- computed considering the sizes of the source table and images and the number
- of segments in the database cluster.
- @note Using the default for 'buffer_size' will produce buffers that are relatively
- large, which generally results in the fastest fit() runtime with Keras. Setting a
- smaller buffer size may cause the preprocessor to run faster (although this is not
- guaranteed, since it depends on database cluster size, data set, and other factors).
- But since preprocessing is usually a one-time operation and fit() is called many times,
- by default buffer sizes are optimized to make fit() as fast as possible. Note that
- specifying a 'buffer_size' does not guarantee that exact value will be used. Actual buffer
- size is adjusted to avoid data skew, which adversely impacts fit() runtime.
- </dd>
-
- <dt>normalizing_const (optional)</dt>
- <dd>REAL, default: 1.0. The normalizing constant to divide
- each value in the 'independent_varname' array by. For example,
- you would use 255 for this value if the image data is in the form 0-255.
- </dd>
-
- <dt>num_classes (optional)</dt>
- <dd>INTEGER, default: NULL. Number of class labels for 1-hot
- encoding. If NULL, the 1-hot encoded array
- length will be equal to the number
- of distinct class values found in the input table.
- </dd>
-
- <dt>distribution_rules (optional)</dt>
- <dd>TEXT, default: 'all_segments'. Specifies how to distribute the
- 'output_table'. This is important for how the fit function will use
- resources on the cluster. The default 'all_segments' means the
- 'output_table' will be distributed to all segments in the database
- cluster.
-
- If you specify 'gpu_segments' then the 'output_table' will be distributed
- to all segments that are on hosts that have GPUs attached. This will make
- maximum use of GPU resources when training a deep learning model.
-
- You can also specify the name of a resources table containing the segments
- to be used for training. This table must contain a column called 'dbid' that
- specifies the segment id from the 'gp_segment_configuration' table [2].
- Refer to the utility function <a href="group__grp__gpu__configuration.html">GPU Configuration</a>
- for more information on how to
- identify segments attached to hosts that are GPU enabled.
- </dd>
-</dl>
-
-@anchor validation_preprocessor_dl
-@par Preprocessor for Validation Image Data
-<pre class="syntax">
-validation_preprocessor_dl(source_table,
- output_table,
- dependent_varname,
- independent_varname,
- training_preprocessor_table,
- buffer_size,
- distribution_rules
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
- <dt>source_table</dt>
- <dd>TEXT. Name of the table containing validation dataset.
- Can also be a view.
- </dd>
-
- <dt>output_table</dt>
- <dd>TEXT. Name of the output table from the validation
- preprocessor which will be used as input to algorithms that support mini-batching.
- The arrays packed into the output table are
- normalized using the same normalizing constant from the
- training preprocessor as specified in
- the 'training_preprocessor_table' parameter described below.
- Validation data is not shuffled.
- For performance reasons,
- packed arrays are converted to PostgreSQL bytea format, which is a
- variable-length binary string.
- </dd>
-
- <dt>dependent_varname</dt>
- <dd>TEXT. Name of the dependent variable column.
- @note The mini-batch preprocessor automatically 1-hot encodes
- dependent variables of all types. The exception is numeric array types
- (integer and float), where we assume these are already 1-hot encoded,
- so these will just be passed through as is.
- </dd>
-
- <dt>independent_varname</dt>
- <dd>TEXT. Name of the independent variable column. The column must be
- a numeric array type.
- </dd>
-
- <dt>training_preprocessor_table</dt>
- <dd>TEXT. The output table obtained by
- running training_preprocessor_dl().
- Validation data is preprocessed in the same way as
- training data, i.e., same normalizing constant and dependent
- variable class values.
- </dd>
-
- <dt>buffer_size (optional)</dt>
- <dd>INTEGER, default: computed. Buffer size is the number of rows from
- the source table that are packed into one row of the preprocessor output
- table. In the case of images, the source table will have one image per row,
- and the output table will have multiple images per row. The default value is
- computed considering the sizes of the source table and images and the number
- of segments in the database cluster.
- @note Using the default for 'buffer_size' will produce buffers that are relatively
- large, which generally results in the fastest fit() runtime with Keras. Setting a
- smaller buffer size may cause the preprocessor to run faster (although this is not
- guaranteed, since it depends on database cluster size, data set, and other factors).
- But since preprocessing is usually a one-time operation and fit() is called many times,
- by default buffer sizes are optimized to make fit() as fast as possible. Note that
- specifying a 'buffer_size' does not guarantee that exact value will be used. Actual buffer
- size is adjusted to avoid data skew, which adversely impacts fit() runtime.
- </dd>
-
- <dt>distribution_rules (optional)</dt>
- <dd>TEXT, default: 'all_segments'. Specifies how to distribute the
- 'output_table'. This is important for how the fit function will use
- resources on the cluster. The default 'all_segments' means the
- 'output_table' will be distributed to all segments in the database
- cluster.
-
- If you specify 'gpu_segments' then the 'output_table' will be distributed
- to all segments that are on hosts that have GPUs attached. This will make
- maximum use of GPU resources when training a deep learning model.
-
- You can also specify the name of a resources table containing the segments
- to be used for training. This table must contain a column called 'dbid' that
- specifies the segment id from the 'gp_segment_configuration' table [2].
- Refer to the utility function <a href="group__grp__gpu__configuration.html">GPU Configuration</a>
- for more information on how to
- identify segments attached to hosts that are GPU enabled.
- </dd>
-
-</dl>
-
-@anchor output
-@par Output Tables
-<br>
- The output tables produced by both training_preprocessor_dl() and
- validation_preprocessor_dl() contain the following columns:
- <table class="output">
- <tr>
- <th>independent_var</th>
- <td>BYTEA. Packed array of independent variables in PostgreSQL bytea format.
- Arrays of independent variables packed into the output table are
- normalized by dividing each element in the independent variable array by the
- optional 'normalizing_const' parameter. Training data is shuffled, but
- validation data is not.
- </td>
- </tr>
- <tr>
- <th>dependent_var</th>
- <td>BYTEA. Packed array of dependent variables in PostgreSQL bytea format.
- The dependent variable is always one-hot encoded as an
- integer array. For now, we are assuming that
- input_preprocessor_dl() will be used
- only for classification problems using deep learning. So
- the dependent variable is one-hot encoded, unless it's already a
- numeric array in which case we assume it's already one-hot
- encoded and just cast it to an integer array.
- </td>
- </tr>
- <tr>
- <th>independent_var_shape</th>
- <td>INTEGER[]. Shape of the independent variable array after preprocessing.
- The first element is the number of images packed per row, and subsequent
- elements will depend on how the image is described (e.g., channels first or last).
- </td>
- </tr>
- <tr>
- <th>dependent_var_shape</th>
- <td>INTEGER[]. Shape of the dependent variable array after preprocessing.
- The first element is the number of images packed per row, and the second
- element is the number of class values.
- </td>
- </tr>
- <tr>
- <th>buffer_id</th>
- <td>INTEGER. Unique id for each row in the packed table.
- </td>
- </tr>
- </table>
-
-A summary table named \<output_table\>_summary is also created, which
-has the following columns (the columns are the same for
-both validation_preprocessor_dl() and training_preprocessor_dl() ):
- <table class="output">
- <tr>
- <th>source_table</th>
- <td>Name of the source table.</td>
- </tr>
- <tr>
- <th>output_table</th>
- <td>Name of output table generated by preprocessor.</td>
- </tr>
- <tr>
- <th>dependent_varname</th>
- <td>Dependent variable from the source table.</td>
- </tr>
- <tr>
- <th>independent_varname</th>
- <td>Independent variable from the source table.</td>
- </tr>
- <tr>
- <th>dependent_vartype</th>
- <td>Type of the dependent variable from the source table.</td>
- </tr>
- <tr>
- <th>class_values</th>
- <td>The dependent level values that one-hot encoding maps to.</td>
- </tr>
- <tr>
- <th>buffer_size</th>
- <td>Buffer size used in preprocessing step.</td>
- </tr>
- <tr>
- <th>normalizing_const</th>
- <td>The value used to normalize the input image data.</td>
- </tr>
- <tr>
- <th>num_classes</th>
- <td>Number of dependent levels the one-hot encoding is created
- for. NULLs are padded at the end if the number of distinct class
- levels found in the input data is less than the 'num_classes' parameter
- specified in training_preprocessor_dl().</td>
- </tr>
- <tr>
- <th>distribution_rules</th>
- <td>This is the list of segment id's in the form of 'dbid'
- describing how the 'output_table' is distributed,
- as per the 'distribution_rules' input parameter.
- If the 'distribution_rules' parameter is set to 'all_segments', then
- this will also be set to 'all_segments'.</td>
- </tr>
- <tr>
- <th>__internal_gpu_config__</th>
- <td>For internal use. (Note: this is the list of segment id's
- where data is distributed in the form of 'content' id, which
- is different from 'dbid' [2].)</td>
- </tr>
- </table>
-
-@anchor example
-@par Examples
--# Create an artificial 2x2 resolution color image data set with 3 possible classifications.
-The RGB values are per-pixel arrays:
-<pre class="example">
-DROP TABLE IF EXISTS image_data;
-CREATE TABLE image_data AS (
- SELECT ARRAY[
- ARRAY[
- ARRAY[(random() * 256)::integer, -- pixel (1,1)
- (random() * 256)::integer,
- (random() * 256)::integer],
- ARRAY[(random() * 256)::integer, -- pixel (2,1)
- (random() * 256)::integer,
- (random() * 256)::integer]
- ],
- ARRAY[
- ARRAY[(random() * 256)::integer, -- pixel (1,2)
- (random() * 256)::integer,
- (random() * 256)::integer],
- ARRAY[(random() * 256)::integer, -- pixel (2,1)
- (random() * 256)::integer,
- (random() * 256)::integer]
- ]
- ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species
- FROM generate_series(1, 52)
-);
-SELECT * FROM image_data;
-</pre>
-<pre class="result">
- rgb | species
---------------------------------------------------------------+---------
- {{{124,198,44},{91,47,130}},{{24,175,69},{196,189,166}}} | dog
- {{{111,202,129},{198,249,254}},{{141,37,88},{187,167,113}}} | dog
- {{{235,53,39},{145,167,209}},{{197,147,222},{55,218,53}}} | dog
- {{{231,48,125},{248,233,151}},{{63,125,230},{33,24,70}}} | dog
- {{{92,146,121},{163,241,110}},{{75,88,72},{218,90,12}}} | bird
- {{{88,114,59},{202,211,152}},{{92,76,58},{77,186,134}}} | dog
- {{{2,96,255},{14,48,19}},{{240,55,115},{137,255,245}}} | dog
- {{{165,122,98},{16,115,240}},{{4,106,116},{108,242,210}}} | dog
- {{{155,207,101},{214,167,24}},{{118,240,228},{199,230,21}}} | dog
- {{{94,212,15},{48,66,170}},{{255,167,128},{166,191,246}}} | dog
- {{{169,69,131},{16,98,225}},{{228,113,17},{38,27,17}}} | bird
- {{{156,183,139},{146,77,46}},{{80,202,230},{146,84,239}}} | dog
- {{{190,210,147},{227,31,66}},{{229,251,84},{51,118,240}}} | bird
- {{{253,175,200},{237,151,107}},{{207,56,162},{133,39,35}}} | cat
- {{{146,185,108},{14,10,105}},{{188,210,86},{83,61,36}}} | dog
- {{{223,169,177},{3,200,250}},{{112,91,16},{193,32,151}}} | cat
- {{{249,145,240},{144,153,58}},{{131,156,230},{56,50,75}}} | dog
- {{{212,186,229},{52,251,197}},{{230,121,201},{35,215,119}}} | cat
- {{{234,94,23},{114,196,94}},{{242,249,90},{223,24,109}}} | bird
- {{{111,36,145},{77,135,123}},{{171,158,237},{111,252,222}}} | dog
- {{{90,74,240},{231,133,95}},{{11,21,173},{146,144,88}}} | cat
- {{{170,52,237},{13,114,71}},{{87,99,46},{220,194,56}}} | bird
- {{{8,17,92},{64,2,203}},{{10,131,145},{4,129,30}}} | cat
- {{{217,218,207},{74,68,186}},{{127,107,76},{38,60,16}}} | bird
- {{{193,34,83},{203,99,58}},{{251,224,50},{228,118,113}}} | dog
- {{{146,218,155},{32,159,243}},{{146,218,189},{101,114,25}}} | bird
- {{{179,160,74},{204,81,246}},{{50,189,39},{60,42,185}}} | cat
- {{{13,82,174},{198,151,84}},{{65,249,100},{179,234,104}}} | cat
- {{{162,190,124},{184,66,138}},{{10,240,80},{161,68,145}}} | dog
- {{{164,144,199},{53,42,111}},{{122,174,128},{220,143,100}}} | cat
- {{{160,138,104},{177,86,3}},{{104,226,149},{181,16,229}}} | dog
- {{{246,119,211},{229,249,119}},{{117,192,172},{159,47,38}}} | cat
- {{{175,1,220},{18,78,124}},{{156,181,45},{242,185,148}}} | bird
- {{{50,113,246},{101,213,180}},{{56,103,151},{87,169,124}}} | cat
- {{{73,109,147},{22,81,197}},{{135,71,42},{91,251,98}}} | bird
- {{{206,61,255},{25,151,211}},{{211,124,7},{206,64,237}}} | cat
- {{{201,71,34},{182,142,43}},{{198,172,171},{230,1,23}}} | bird
- {{{142,158,2},{223,45,205}},{{118,177,223},{232,178,141}}} | cat
- {{{86,190,128},{195,172,14}},{{97,173,237},{142,123,99}}} | cat
- {{{26,72,148},{79,226,156}},{{96,62,220},{99,9,230}}} | bird
- {{{154,234,103},{184,18,65}},{{146,225,139},{214,156,10}}} | cat
- {{{244,169,103},{218,143,2}},{{196,246,186},{214,55,76}}} | bird
- {{{20,226,7},{96,153,200}},{{130,236,147},{229,38,142}}} | bird
- {{{172,102,107},{50,11,109}},{{145,9,123},{193,28,107}}} | bird
- {{{143,243,247},{132,104,137}},{{94,3,169},{253,246,59}}} | bird
- {{{78,74,228},{51,200,218}},{{170,155,190},{164,18,51}}} | dog
- {{{163,226,161},{56,182,239}},{{129,154,35},{73,116,205}}} | bird
- {{{74,243,3},{172,182,149}},{{101,34,163},{111,138,95}}} | cat
- {{{224,178,126},{4,61,93}},{{174,238,96},{118,232,208}}} | bird
- {{{55,236,249},{7,189,242}},{{151,173,130},{49,232,5}}} | bird
- {{{9,16,30},{128,32,85}},{{108,25,91},{41,11,243}}} | bird
- {{{141,35,191},{146,240,141}},{{207,239,166},{102,194,121}}} | bird
-(52 rows)
-</pre>
--# Run the preprocessor for training image data:
-<pre class="example">
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data', -- Source table
- 'image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- NULL, -- Buffer size
- 255 -- Normalizing constant
- );
-</pre>
-For small datasets like in this example, buffer size is mainly determined
-by the number of segments in the database. For a Greenplum database with 2 segments,
-there will be 2 rows with a buffer size of 26. For PostgresSQL, there would
-be only one row with a buffer size of 52 since it is a single node database.
-For larger data sets, other factors go into computing buffers size besides
-number of segments.
-Here is the packed output table of training data for our simple example:
-<pre class="example">
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-</pre>
-<pre class="result">
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,2,2,3} | {26,3} | 0
- {26,2,2,3} | {26,3} | 1
-(2 rows)
-</pre>
-Review the output summary table:
-<pre class="example">
-\\x on
-SELECT * FROM image_data_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-----------+------------------
-source_table | image_data
-output_table | image_data_packed
-dependent_varname | species
-independent_varname | rgb
-dependent_vartype | text
-class_values | {bird,cat,dog}
-buffer_size | 26
-normalizing_const | 255
-num_classes | 3
-distribution_rules | all_segments
-__internal_gpu_config__ | all_segments
-</pre>
-
--# Run the preprocessor for the validation dataset.
-In this example, we use the same images for
-validation to demonstrate, but normally validation data
-is different than training data:
-<pre class="example">
-DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;
-SELECT madlib.validation_preprocessor_dl(
- 'image_data', -- Source table
- 'val_image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- 'image_data_packed', -- From training preprocessor step
- NULL -- Buffer size
- );
-</pre>
-We could choose to use a different buffer size compared to the
-training_preprocessor_dl run (but generally don't need to).
-Other parameters such as num_classes and
-normalizing_const that were passed to training_preprocessor_dl are
-automatically inferred using the image_data_packed param that is passed.
-Here is the packed output table of validation data for our simple example:
-<pre class="example">
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM val_image_data_packed ORDER BY buffer_id;
-</pre>
-<pre class="result">
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,2,2,3} | {26,3} | 0
- {26,2,2,3} | {26,3} | 1
-(2 rows)
-</pre>
-Review the output summary table:
-<pre class="example">
-\\x on
-SELECT * FROM val_image_data_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-----------+----------------------
-source_table | image_data
-output_table | val_image_data_packed
-dependent_varname | species
-independent_varname | rgb
-dependent_vartype | text
-class_values | {bird,cat,dog}
-buffer_size | 26
-normalizing_const | 255
-num_classes | 3
-distribution_rules | all_segments
-__internal_gpu_config__ | all_segments
-</pre>
-
--# Load data in another format. Create an artificial 2x2 resolution color image
-data set with 3 possible classifications. The RGB values are unrolled into a flat array:
-<pre class="example">
-DROP TABLE IF EXISTS image_data;
-CREATE TABLE image_data AS (
-SELECT ARRAY[
- (random() * 256)::integer, -- R values
- (random() * 256)::integer,
- (random() * 256)::integer,
- (random() * 256)::integer,
- (random() * 256)::integer, -- G values
- (random() * 256)::integer,
- (random() * 256)::integer,
- (random() * 256)::integer,
- (random() * 256)::integer, -- B values
- (random() * 256)::integer,
- (random() * 256)::integer,
- (random() * 256)::integer
- ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species
-FROM generate_series(1, 52)
-);
-SELECT * FROM image_data;
-</pre>
-<pre class="result">
- rgb | species
---------------------------------------------------+---------
- {26,150,191,113,235,57,145,143,44,145,85,25} | dog
- {240,43,225,15,220,136,186,209,49,130,55,111} | bird
- {25,191,37,77,193,62,249,228,97,33,81,7} | cat
- {141,223,46,195,201,19,207,78,160,130,157,89} | cat
- {39,249,168,164,223,193,99,4,14,37,66,7} | cat
- {159,250,127,44,151,254,11,211,247,137,79,233} | cat
- {19,230,76,253,42,175,230,143,184,133,27,215} | cat
- {199,224,144,5,64,19,200,186,109,218,108,70} | bird
- {148,136,4,41,185,104,203,253,113,151,166,76} | bird
- {230,132,114,213,210,139,91,199,240,142,203,75} | bird
- {166,188,96,217,135,70,93,249,27,47,132,118} | bird
- {118,120,222,236,110,83,240,47,19,206,222,51} | bird
- {230,3,26,47,93,144,167,59,123,21,142,107} | cat
- {250,224,62,136,112,142,88,187,24,1,168,216} | bird
- {52,144,231,12,76,1,162,11,114,141,69,3} | cat
- {166,172,246,169,200,102,62,57,239,75,165,88} | dog
- {151,50,112,227,199,97,47,4,43,123,116,133} | bird
- {39,185,96,127,80,248,177,191,218,120,32,9} | dog
- {25,172,34,34,40,109,166,23,60,216,246,54} | bird
- {163,39,89,170,95,230,137,141,169,82,159,121} | dog
- {131,143,183,138,151,90,177,240,4,16,214,141} | dog
- {99,233,100,9,159,140,30,202,29,169,120,62} | bird
- {99,162,69,10,204,169,219,20,106,170,111,16} | bird
- {16,246,27,32,187,226,0,75,231,64,94,175} | bird
- {25,135,244,101,50,4,91,77,36,22,47,37} | dog
- {22,101,191,197,96,138,78,198,155,138,193,51} | bird
- {236,22,110,30,181,20,218,21,236,97,91,73} | dog
- {160,57,34,212,239,197,233,174,164,97,88,153} | cat
- {226,170,192,123,242,224,190,51,163,192,91,105} | bird
- {149,174,12,72,112,1,37,153,118,201,79,121} | bird
- {34,250,232,222,218,221,234,201,138,66,186,58} | bird
- {162,55,85,159,247,234,77,3,50,189,4,87} | dog
- {122,32,164,243,0,198,237,232,164,199,197,142} | dog
- {80,209,75,138,169,236,193,254,140,184,232,217} | bird
- {112,148,114,137,13,107,105,75,243,218,218,75} | dog
- {241,76,61,202,76,112,90,51,125,166,52,30} | bird
- {75,132,239,207,49,224,250,19,238,214,154,169} | dog
- {203,43,222,58,231,5,243,71,131,67,63,52} | cat
- {229,12,133,142,179,80,185,145,138,160,149,125} | bird
- {64,251,61,153,13,100,145,181,8,112,118,107} | dog
- {128,223,60,248,126,124,243,188,20,0,31,166} | bird
- {39,22,43,146,138,174,33,65,56,184,155,234} | dog
- {177,247,133,154,159,37,148,30,81,43,29,92} | bird
- {56,127,199,118,105,120,109,239,18,12,20,166} | cat
- {101,209,72,193,207,91,166,27,88,209,203,62} | dog
- {131,195,122,90,18,178,217,217,40,66,81,149} | cat
- {203,137,103,17,60,251,152,64,36,81,168,239} | cat
- {239,97,10,20,194,32,121,129,228,217,11,50} | dog
- {117,4,193,192,223,176,33,232,196,226,8,61} | dog
- {162,21,190,223,120,170,245,230,200,170,250,163} | bird
- {32,67,65,195,2,39,198,28,86,35,172,254} | dog
- {39,19,236,146,87,140,203,121,96,187,62,73} | dog
-(52 rows)
-</pre>
-
--# Run the preprocessor for training image data:
-<pre class="example">
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data', -- Source table
- 'image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- NULL, -- Buffer size
- 255 -- Normalizing constant
- );
-</pre>
-Here is a sample of the packed output table:
-<pre class="example">
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-</pre>
-<pre class="result">
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,12} | {26,3} | 0
- {26,12} | {26,3} | 1
-(2 rows)
-</pre>
-
--# Run the preprocessor for the validation dataset.
-In this example, we use the same images for
-validation to demonstrate, but normally validation data
-is different than training data:
-<pre class="example">
-DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;
-SELECT madlib.validation_preprocessor_dl(
- 'image_data', -- Source table
- 'val_image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- 'image_data_packed', -- From training preprocessor step
- NULL -- Buffer size
- );
-</pre>
-Here is a sample of the packed output summary table:
-<pre class="example">
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM val_image_data_packed ORDER BY buffer_id;
-</pre>
-<pre class="result">
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,12} | {26,3} | 0
- {26,12} | {26,3} | 1
-(2 rows)
-</pre>
-
--# Generally the default buffer size will work well,
-but if you have occasion to change it:
-<pre class="example">
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data', -- Source table
- 'image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- 10, -- Buffer size
- 255 -- Normalizing constant
- );
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-</pre>
-<pre class="result">
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {8,12} | {8,3} | 0
- {9,12} | {9,3} | 1
- {9,12} | {9,3} | 2
- {9,12} | {9,3} | 3
- {9,12} | {9,3} | 4
- {8,12} | {8,3} | 5
-(6 rows)
-</pre>
-Review the output summary table:
-<pre class="example">
-\\x on
-SELECT * FROM image_data_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-----------+------------------
-source_table | image_data
-output_table | image_data_packed
-dependent_varname | species
-independent_varname | rgb
-dependent_vartype | text
-class_values | {bird,cat,dog}
-buffer_size | 10
-normalizing_const | 255
-num_classes | 3
-distribution_rules | all_segments
-__internal_gpu_config__ | all_segments
-</pre>
-
--# Run the preprocessor for image data with num_classes greater than 3 (distinct class values found in table):
-<pre class="example">
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data', -- Source table
- 'image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- NULL, -- Buffer size
- 255, -- Normalizing constant
- 5 -- Number of desired class values
- );
-</pre>
-Here is a sample of the packed output table with the padded 1-hot vector:
-<pre class="example">
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-</pre>
-<pre class="result">
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,12} | {26,5} | 0
- {26,12} | {26,5} | 1
-(2 rows)
-</pre>
-Review the output summary table:
-<pre class="example">
-\\x on
-SELECT * FROM image_data_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-----------+-------------------------
-source_table | image_data
-output_table | image_data_packed
-dependent_varname | species
-independent_varname | rgb
-dependent_vartype | text
-class_values | {bird,cat,dog,NULL,NULL}
-buffer_size | 26
-normalizing_const | 255
-num_classes | 5
-distribution_rules | all_segments
-__internal_gpu_config__ | all_segments
-</pre>
-
--# Using distribution rules to specify how to distribute
-the 'output_table'. This is important for how the fit function
-will use resources on the cluster. To distribute to all segments
-on hosts with GPUs attached:
-<pre class="example">
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data', -- Source table
- 'image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- NULL, -- Buffer size
- 255, -- Normalizing constant
- NULL, -- Number of classes
- 'gpu_segments' -- Distribution rules
- );
-\\x on
-SELECT * FROM image_data_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-----------+------------------
-source_table | image_data
-output_table | image_data_packed
-dependent_varname | species
-independent_varname | rgb
-dependent_vartype | text
-class_values | {bird,cat,dog}
-buffer_size | 26
-normalizing_const | 255
-num_classes | 3
-distribution_rules | {2,3,4,5}
-__internal_gpu_config__ | {0,1,2,3}
-</pre>
-To distribute to only specified segments, create a
-distribution table with a column called 'dbid' that
-lists the segments you want:
-<pre class="example">
-DROP TABLE IF EXISTS segments_to_use;
-CREATE TABLE segments_to_use(
- dbid INTEGER,
- hostname TEXT
-);
-INSERT INTO segments_to_use VALUES
-(2, 'hostname-01'),
-(3, 'hostname-01');
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data', -- Source table
- 'image_data_packed', -- Output table
- 'species', -- Dependent variable
- 'rgb', -- Independent variable
- NULL, -- Buffer size
- 255, -- Normalizing constant
- NULL, -- Number of classes
- 'segments_to_use' -- Distribution rules
- );
-\\x on
-SELECT * FROM image_data_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-----------+------------------
-source_table | image_data
-output_table | image_data_packed
-dependent_varname | species
-independent_varname | rgb
-dependent_vartype | text
-class_values | {bird,cat,dog}
-buffer_size | 26
-normalizing_const | 255
-num_classes | 3
-distribution_rules | {2,3}
-__internal_gpu_config__ | {0,1}
-</pre>
-
-@anchor references
-@par References
-
-[1] "Neural Networks for Machine Learning", Lectures 6a and 6b on mini-batch gradient descent,
-Geoffrey Hinton with Nitish Srivastava and Kevin Swersky, http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf
-
-[2] Greenplum 'gp_segment_configuration' table https://gpdb.docs.pivotal.io/latest/ref_guide/system_catalogs/gp_segment_configuration.html
-
-@anchor related
-@par Related Topics
-
-training_preprocessor_dl()
-
-validation_preprocessor_dl()
-
-gpu_configuration()
-
-*/
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- training_preprocessor_table VARCHAR,
- buffer_size INTEGER,
- distribution_rules TEXT
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- from utilities.control import MinWarning
- with AOControl(False):
- with MinWarning('error'):
- validation_preprocessor_obj = input_data_preprocessor.ValidationDataPreprocessorDL(**globals())
- validation_preprocessor_obj.validation_preprocessor_dl()
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- training_preprocessor_table VARCHAR,
- buffer_size INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.validation_preprocessor_dl($1, $2, $3, $4, $5, $6, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- training_preprocessor_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.validation_preprocessor_dl($1, $2, $3, $4, $5, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl(
- message VARCHAR
-) RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- return input_data_preprocessor.InputDataPreprocessorDocumentation.validation_preprocessor_dl_help(schema_madlib, message)
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl()
-RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- return input_data_preprocessor.InputDataPreprocessorDocumentation.validation_preprocessor_dl_help(schema_madlib, '')
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
--------------------------------------------------------------------------------
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- buffer_size INTEGER,
- normalizing_const REAL,
- num_classes INTEGER,
- distribution_rules TEXT
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- from utilities.control import MinWarning
- with AOControl(False):
- with MinWarning('error'):
- training_preprocessor_obj = input_data_preprocessor.TrainingDataPreprocessorDL(**globals())
- training_preprocessor_obj.training_preprocessor_dl()
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- buffer_size INTEGER,
- normalizing_const REAL,
- num_classes INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, $6, $7, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- buffer_size INTEGER,
- normalizing_const REAL
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, $6, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR,
- buffer_size INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, 1.0, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- source_table VARCHAR,
- output_table VARCHAR,
- dependent_varname VARCHAR,
- independent_varname VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, NULL, 1.0, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl(
- message VARCHAR
-) RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- return input_data_preprocessor.InputDataPreprocessorDocumentation.training_preprocessor_dl_help(schema_madlib, message)
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl()
-RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, input_data_preprocessor)
- return input_data_preprocessor.InputDataPreprocessorDocumentation.training_preprocessor_dl_help(schema_madlib, '')
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
--- aggregation for independent var
-DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(REAL[]);
-CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(REAL[]) (
- SFUNC = array_cat,
- PREFUNC = array_cat,
- STYPE = REAL[]
- );
-
--- aggregation for dependent var
-DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(SMALLINT[]);
-CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(SMALLINT[]) (
- SFUNC = array_cat,
- PREFUNC = array_cat,
- STYPE = SMALLINT[]
- );
-
-CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var REAL[])
-RETURNS BYTEA
-AS
-$$
-import numpy as np
-
-return np.array(var, dtype=np.float32).tobytes()
-$$ LANGUAGE plpythonu;
-
-CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var SMALLINT[])
-RETURNS BYTEA
-AS
-$$
-import numpy as np
-
-return np.array(var, dtype=np.int16).tobytes()
-$$ LANGUAGE plpythonu;
-
-
-CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_real_array(var BYTEA)
-RETURNS REAL[]
-AS
-$$
-import numpy as np
-
-return np.frombuffer(var, dtype=np.float32)
-$$ LANGUAGE plpythonu;
-
-
-CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_smallint_array(var BYTEA)
-RETURNS SMALLINT[]
-AS
-$$
-import numpy as np
-
-return np.frombuffer(var, dtype=np.int16)
-$$ LANGUAGE plpythonu;
-
diff --git a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.py_in b/src/ports/postgres/modules/deep_learning/keras_model_arch_table.py_in
deleted file mode 100644
index 0d6fc7c..0000000
--- a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.py_in
+++ /dev/null
@@ -1,231 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-@file keras_model_arch_table.py_in
-
-@brief keras model arch table management helper functions
-
-@namespace keras_model_arch_table
-"""
-
-from internal.db_utils import quote_literal
-import plpy
-from utilities.control import MinWarning
-from utilities.utilities import get_col_name_type_sql_string
-from utilities.utilities import unique_string
-from utilities.validate_args import columns_missing_from_table
-from utilities.validate_args import input_tbl_valid
-from utilities.validate_args import quote_ident
-from utilities.validate_args import table_exists
-
-class ModelArchSchema:
- """Expected format of keras_model_arch_table.
- Example uses:
-
- from utilities.validate_args import columns_missing_from_table
- from keras_model_arch_table import Format
-
- # Validate names in cols list against actual table
- missing_cols = columns_missing_from_table('my_arch_table', Format.col_names)
-
- # Get model arch from keras model arch table, without hard coding column names
- sql = "SELECT {arch} FROM {table} WHERE {id} = {my_id}"
- .format(arch=Format.model_arch,
- table='my_arch_table',
- id=Format.model_id,
- my_id=1)
- arch = plpy.execute(sql)[0]
-
- """
- col_names = ('model_id', 'model_arch', 'model_weights', 'name', 'description',
- '__internal_madlib_id__')
- col_types = ('SERIAL PRIMARY KEY', 'JSON', 'bytea', 'TEXT', 'TEXT', 'TEXT')
- (MODEL_ID, MODEL_ARCH, MODEL_WEIGHTS, NAME, DESCRIPTION,
- __INTERNAL_MADLIB_ID__) = col_names
-
-@MinWarning("error")
-def load_keras_model(keras_model_arch_table, model_arch, model_weights,
- name, description, **kwargs):
- model_arch_table = quote_ident(keras_model_arch_table)
- if not table_exists(model_arch_table):
- col_defs = get_col_name_type_sql_string(ModelArchSchema.col_names,
- ModelArchSchema.col_types)
-
- sql = "CREATE TABLE {model_arch_table} ({col_defs})" \
- .format(**locals())
-
- plpy.execute(sql, 0)
- plpy.info("Keras Model Arch: Created new keras model architecture table {0}." \
- .format(model_arch_table))
- else:
- missing_cols = columns_missing_from_table(model_arch_table,
- ModelArchSchema.col_names)
- if len(missing_cols) > 0:
- plpy.error("Keras Model Arch: Invalid keras model architecture table {0},"
- " missing columns: {1}".format(model_arch_table,
- missing_cols))
-
- unique_str = unique_string(prefix_has_temp=False)
- insert_query = plpy.prepare("INSERT INTO {model_arch_table} "
- "VALUES(DEFAULT, $1, $2, $3, $4, $5);".format(**locals()),
- ModelArchSchema.col_types[1:])
- insert_res = plpy.execute(insert_query,[model_arch, model_weights, name, description,
- unique_str], 0)
-
- select_query = """SELECT {model_id_col}, {model_arch_col} FROM {model_arch_table}
- WHERE {internal_id_col} = '{unique_str}'""".format(
- model_id_col=ModelArchSchema.MODEL_ID,
- model_arch_col=ModelArchSchema.MODEL_ARCH,
- model_arch_table=model_arch_table,
- internal_id_col=ModelArchSchema.__INTERNAL_MADLIB_ID__,
- unique_str=unique_str)
- select_res = plpy.execute(select_query,1)
-
- plpy.info("Keras Model Arch: Added model id {0} to {1} table".
- format(select_res[0][ModelArchSchema.MODEL_ID], model_arch_table))
-
-@MinWarning("error")
-def delete_keras_model(keras_model_arch_table, model_id, **kwargs):
- model_arch_table = quote_ident(keras_model_arch_table)
- input_tbl_valid(model_arch_table, "Keras Model Arch")
-
- missing_cols = columns_missing_from_table(model_arch_table, ModelArchSchema.col_names)
- if len(missing_cols) > 0:
- plpy.error("Keras Model Arch: Invalid keras model architecture table {0},"
- " missing columns: {1}".format(model_arch_table, missing_cols))
-
- sql = """
- DELETE FROM {model_arch_table} WHERE {model_id_col}={model_id}
- """.format(model_arch_table=model_arch_table, model_id_col=ModelArchSchema.MODEL_ID,
- model_id=model_id)
- res = plpy.execute(sql, 0)
-
- if res.nrows() > 0:
- plpy.info("Keras Model Arch: Model id {0} has been deleted from {1}.".
- format(model_id, model_arch_table))
- else:
- plpy.error("Keras Model Arch: Model id {0} not found".format(model_id))
-
- sql = "SELECT {0} FROM {1}".format(ModelArchSchema.MODEL_ID, model_arch_table)
- res = plpy.execute(sql, 0)
- if not res:
- plpy.info("Keras Model Arch: Dropping empty keras model architecture "\
- "table {model_arch_table}".format(model_arch_table=model_arch_table))
- sql = "DROP TABLE {0}".format(model_arch_table)
- plpy.execute(sql, 0)
-
-class KerasModelArchDocumentation:
- @staticmethod
- def _returnHelpMsg(schema_madlib, message, summary, usage, method):
- if not message:
- return summary
- elif message.lower() in ('usage', 'help', '?'):
- return usage
- return """
- No such option. Use "SELECT {schema_madlib}.{method}()"
- for help.
- """.format(**locals())
-
- @staticmethod
- def load_keras_model_help(schema_madlib, message):
- method = "load_keras_model"
- summary = """
- ----------------------------------------------------------------
- SUMMARY
- ----------------------------------------------------------------
- The architecture of the model to be used in madlib_keras_train()
- function must be stored in a table, the details of which must be
- provided as parameters to the madlib_keras_train module. This is
- a helper function to help users insert JSON blobs of Keras model
- architectures into a table.
- If the output table already exists, the model_arch specified will
- be added as a new row into the table. The output table could thus
- act as a repository of Keras model architectures.
-
- For more details on function usage:
- SELECT {schema_madlib}.{method}('usage')
- """.format(**locals())
-
- usage = """
- ---------------------------------------------------------------------------
- USAGE
- ---------------------------------------------------------------------------
- SELECT {schema_madlib}.{method}(
- keras_model_arch_table, -- Output table to load keras model arch.
- model_arch, -- JSON of the model architecture to insert.
- model_weights, -- Model weights to load as a PostgreSQL
- binary data type.
- name, -- Free text string to identify a name
- description -- Free text string to provide a description
- );
-
-
- ---------------------------------------------------------------------------
- OUTPUT
- ---------------------------------------------------------------------------
- The output table produced by load_keras_model contains the following columns:
-
- 'model_id' -- SERIAL PRIMARY KEY. Model ID.
- 'model_arch' -- JSON. JSON blob of the model architecture.
- 'model_weights' -- bytea. weights of the model for warm start.
- '__internal_madlib_id__' -- TEXT. Unique id for model arch.
-
- """.format(**locals())
-
- return KerasModelArchDocumentation._returnHelpMsg(
- schema_madlib, message, summary, usage, method)
-# ---------------------------------------------------------------------
-
- @staticmethod
- def delete_keras_model_help(schema_madlib, message):
- method = "delete_keras_model"
- summary = """
- ----------------------------------------------------------------
- SUMMARY
- ----------------------------------------------------------------
- Delete the model architecture corresponding to the provided model_id
- from the model architecture repository table (keras_model_arch_table).
-
- For more details on function usage:
- SELECT {schema_madlib}.{method}('usage')
- """.format(**locals())
-
- usage = """
- ---------------------------------------------------------------------------
- USAGE
- ---------------------------------------------------------------------------
- SELECT {schema_madlib}.{method}(
- keras_model_arch_table VARCHAR, -- Table containing Keras model architectures.
- model_id INTEGER -- The id of the model arch to be deleted.
- );
-
-
- ---------------------------------------------------------------------------
- OUTPUT
- ---------------------------------------------------------------------------
- This method deletes the row corresponding to the given model_id in
- keras_model_arch_table. This also tries to drop the table if the table is
- empty after dropping the model_id. If there are any views depending on the
- table, a warning message is displayed and the table is not dropped.
-
- ---------------------------------------------------------------------------
- """.format(**locals())
-
- return KerasModelArchDocumentation._returnHelpMsg(
- schema_madlib, message, summary, usage, method)
diff --git a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.sql_in b/src/ports/postgres/modules/deep_learning/keras_model_arch_table.sql_in
deleted file mode 100644
index cc915bb..0000000
--- a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.sql_in
+++ /dev/null
@@ -1,429 +0,0 @@
-/* ----------------------------------------------------------------------- *//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *
- * @file model_arch_table.sql_in
- *
- * @brief SQL functions for multilayer perceptron
- * @date June 2012
- *
- *
- *//* ----------------------------------------------------------------------- */
-
-m4_include(`SQLCommon.m4')
-/**
-@addtogroup grp_keras_model_arch
-
-@brief Utility function to load model architectures and weights into a table.
-
-\warning <em> This MADlib method is still in early stage development.
-Interface and implementation are subject to change. </em>
-
-<div class="toc"><b>Contents</b><ul>
-<li class="level1"><a href="#load_keras_model">Load Model</a></li>
-<li class="level1"><a href="#delete_keras_model">Delete Model</a></li>
-<li class="level1"><a href="#example">Examples</a></li>
-<li class="level1"><a href="#related">Related Topics</a></li>
-</ul></div>
-
-This utility function loads model architectures and
-weights into a table for use by deep learning algorithms.
-Model architecture is in JSON form
-and model weights are in the form of PostgreSQL binary data types (bytea).
-If the output table already exists, a new row is inserted
-into the table so it can act as a repository for multiple model
-architectures and weights.
-
-There is also a utility function to delete a model
-from the table.
-
-@anchor load_keras_model
-@par Load Model
-
-<pre class="syntax">
-load_keras_model(
- keras_model_arch_table,
- model_arch,
- model_weights,
- name,
- description
- )
-</pre>
-\b Arguments
-<dl class="arglist">
- <dt>keras_model_arch_table</dt>
- <dd>VARCHAR. Output table to load keras model architecture and weights.
- </dd>
-
- <dt>model_arch</dt>
- <dd>JSON. JSON of the model architecture to load.
- </dd>
-
- <dt>model_weights (optional)</dt>
- <dd>bytea. Model weights to load as a PostgreSQL
- binary data type.
- </dd>
-
- <dt>name (optional)</dt>
- <dd>TEXT, default: NULL. Free text string to provide
- a name, if desired.
- </dd>
-
- <dt>description (optional)</dt>
- <dd>TEXT, default: NULL. Free text string to provide
- a description, if desired.
- </dd>
-
-</dl>
-
-<b>Output table</b>
-<br>
- The output table contains the following columns:
- <table class="output">
- <tr>
- <th>model_id</th>
- <td>SERIAL PRIMARY KEY. Model ID.
- </td>
- </tr>
- <tr>
- <th>model_arch</th>
- <td>JSON. JSON blob of the model architecture.
- </td>
- </tr>
- <tr>
- <th>model_weights</th>
- <td>BYTEA. Weights of the model which may be used for warm start
- or transfer learning.
- Weights are stored as a PostgreSQL binary data type.
- </td>
- </tr>
- <tr>
- <th>name</th>
- <td>TEXT. Name of model (free text).
- </td>
- </tr>
- <tr>
- <th>description</th>
- <td>TEXT. Description of model (free text).
- </td>
- </tr>
- <tr>
- <th>__internal_madlib_id__</th>
- <td>TEXT. Unique id for model arch. This is an id used internally be MADlib.
- </td>
- </tr>
- </table>
-</br>
-
-@anchor delete_keras_model
-@par Delete Model
-
-<pre class="syntax">
-delete_keras_model(
- keras_model_arch_table
- model_id
-)
-</pre>
-\b Arguments
-<dl class="arglist">
- <dt>keras_model_arch_table</dt>
- <dd>VARCHAR. Table containing model architectures and weights.
- </dd>
-
- <dt>model_id</dt>
- <dd>INTEGER. The id of the model to be deleted.
- </dd>
-</dl>
-
-@anchor example
-@par Examples
--# Define model architecture. Use Keras to define
-the model architecture:
-<pre class="example">
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model_simple = Sequential()
-model_simple.add(Dense(10, activation='relu', input_shape=(4,)))
-model_simple.add(Dense(10, activation='relu'))
-model_simple.add(Dense(3, activation='softmax'))
-model_simple.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type) Output Shape Param #
-=================================================================
-dense_1 (Dense) (None, 10) 50
-_________________________________________________________________
-dense_2 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_3 (Dense) (None, 3) 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-</pre>
-Export the model to JSON:
-<pre class="example">
-model_simple.to_json()
-</pre>
-<pre class="result">
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-</pre>
--# Load into model architecture table:
-<pre class="example">
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Sophie', -- Name
- 'A simple model' -- Descr
-);
-SELECT COUNT(*) FROM model_arch_library;
-</pre>
-<pre class="result">
- count
--------+
- 1
-</pre>
-Load another model architecture:
-<pre class="example">
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Maria', -- Name
- 'Also a simple model' -- Descr
-);
-SELECT COUNT(*) FROM model_arch_library;
-</pre>
-<pre class="result">
- count
--------+
- 2
-</pre>
--# Load model weights. To load weights from previous
-MADlib run, use UPDATE to load directly into the table.
-For example, if 'model_weights' are the weights in the
-output table 'iris_model' from a previous run
-of 'madlib_keras_fit()' :
-<pre class="example">
-UPDATE model_arch_library SET model_weights = model_weights FROM iris_model WHERE model_id = 2;
-</pre>
-To load weights from Keras using a PL/Python function,
-we need to flatten then serialize the weights to store
-as a PostgreSQL binary data type. Byte format is more
-efficient on space and memory compared to a numeric array.
-The model weights will be de-serialized when
-passed to Keras functions.
-<pre class="example">
-CREATE OR REPLACE FUNCTION load_weights() RETURNS VOID AS
-$$
-from keras.layers import *
-from keras import Sequential
-import numpy as np
-import plpy
-\#
-\# create model
-model = Sequential()
-model.add(Dense(10, activation='relu', input_shape=(4,)))
-model.add(Dense(10, activation='relu'))
-model.add(Dense(3, activation='softmax'))
-\#
-\# get weights, flatten and serialize
-weights = model.get_weights()
-weights_flat = [w.flatten() for w in weights]
-weights1d = np.concatenate(weights_flat).ravel()
-weights_bytea = weights1d.tostring()
-\#
-\# load query
-load_query = plpy.prepare("""SELECT madlib.load_keras_model(
- 'model_arch_library',
- $1, $2)
- """, ['json','bytea'])
-plpy.execute(load_query, [model.to_json(), weights_bytea])
-$$ language plpythonu;
--- Call load function
-SELECT load_weights();
--- Check weights loaded OK
-SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;
-</pre>
-<pre class="result">
- count
--------+
- 1
-</pre>
-Load weights from Keras using psycopg2. (Psycopg is a PostgreSQL database adapter for the
-Python programming language.) As above we need to flatten then serialize the weights to store as a
-PostgreSQL binary data type. Note that the psycopg2.Binary function used below will increase the size of the
-Python object for the weights, so if your model is large it might be better to use a PL/Python function as above.
-<pre class="example">
-import psycopg2
-import psycopg2 as p2
-conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')
-cur = conn.cursor()
-from keras.layers import *
-from keras import Sequential
-import numpy as np
-\#
-\# create model
-model = Sequential()
-model.add(Dense(10, activation='relu', input_shape=(4,)))
-model.add(Dense(10, activation='relu'))
-model.add(Dense(3, activation='softmax'))
-\#
-\# get weights, flatten and serialize
-weights = model.get_weights()
-weights_flat = [w.flatten() for w in weights]
-weights1d = np.concatenate(weights_flat).ravel()
-weights_bytea = psycopg2.Binary(weights1d.tostring())
-\#
-\# load query
-query = "SELECT madlib.load_keras_model('model_arch_library', %s,%s)"
-cur.execute(query,[model.to_json(),weights_bytea])
-conn.commit()
-</pre>
-From SQL check if weights loaded OK:
-<pre class="example">
-SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;
-</pre>
-<pre class="result">
- count
--------+
- 2
-</pre>
--# Delete one of the models:
-<pre class="example">
-SELECT madlib.delete_keras_model('model_arch_library', -- Output table
- 1 -- Model id
- );
-SELECT COUNT(*) FROM model_arch_library;
-</pre>
-<pre class="result">
- count
--------+
- 3
-</pre>
-
-@anchor related
-@par Related Topics
-
-See keras_model_arch_table.sql_in
-
-*/
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model(
- keras_model_arch_table VARCHAR,
- model_arch JSON,
- model_weights bytea,
- name TEXT,
- description TEXT
-)
- RETURNS VOID AS $$
- PythonFunctionBodyOnlyNoSchema(`deep_learning', `keras_model_arch_table')
- from utilities.control import AOControl
- with AOControl(False):
- keras_model_arch_table.load_keras_model(**globals())
-$$ LANGUAGE plpythonu VOLATILE;
-
--- Function to add a keras model to arch table
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model(
- keras_model_arch_table VARCHAR,
- model_arch JSON
-)
-RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.load_keras_model($1, $2, NULL, NULL, NULL)
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model(
- keras_model_arch_table VARCHAR,
- model_arch JSON,
- model_weights bytea
-)
- RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.load_keras_model($1, $2, $3, NULL, NULL)
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model(
- keras_model_arch_table VARCHAR,
- model_arch JSON,
- model_weights bytea,
- name TEXT
-)
- RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.load_keras_model($1, $2, $3, $4, NULL)
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-
-
-
-
-
--- Functions for online help
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model(
- message VARCHAR
-) RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, keras_model_arch_table)
- return keras_model_arch_table.KerasModelArchDocumentation.load_keras_model_help(schema_madlib, message)
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model()
-RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, keras_model_arch_table)
- return keras_model_arch_table.KerasModelArchDocumentation.load_keras_model_help(schema_madlib, '')
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
--- Function to delete a keras model from arch table
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.delete_keras_model(
- keras_model_arch_table VARCHAR,
- model_id INTEGER
-)
-RETURNS VOID AS $$
- PythonFunctionBodyOnlyNoSchema(`deep_learning',`keras_model_arch_table')
- from utilities.control import AOControl
- with AOControl(False):
- keras_model_arch_table.delete_keras_model(**globals())
-$$ LANGUAGE plpythonu VOLATILE;
-
--- Functions for online help
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.delete_keras_model(
- message VARCHAR
-) RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, keras_model_arch_table)
- return keras_model_arch_table.KerasModelArchDocumentation.delete_keras_model_help(schema_madlib, message)
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.delete_keras_model()
-RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(deep_learning, keras_model_arch_table)
- return keras_model_arch_table.KerasModelArchDocumentation.delete_keras_model_help(schema_madlib, '')
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
deleted file mode 100644
index ee27554..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
+++ /dev/null
@@ -1,944 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import datetime
-import os
-import plpy
-import sys
-import time
-
-from keras import backend as K
-from keras.layers import *
-from keras.models import *
-from keras.optimizers import *
-from keras.regularizers import *
-from madlib_keras_helper import *
-from madlib_keras_validator import *
-from madlib_keras_wrapper import *
-from model_arch_info import *
-
-from utilities.utilities import _assert
-from utilities.utilities import add_postfix
-from utilities.utilities import is_platform_pg
-from utilities.utilities import get_segments_per_host
-from utilities.utilities import get_seg_number
-from utilities.utilities import madlib_version
-from utilities.utilities import unique_string
-from utilities.validate_args import get_expr_type
-from utilities.control import MinWarning
-import tensorflow as tf
-
-class SD_STORE:
- SESS = 'sess'
- SEGMENT_MODEL = 'segment_model'
-
- @staticmethod
- def init_SD(SD, sess, segment_model):
- SD[SD_STORE.SESS] = sess
- SD[SD_STORE.SEGMENT_MODEL] = segment_model
-
- @staticmethod
- def clear_SD(SD):
- del SD[SD_STORE.SEGMENT_MODEL]
- del SD[SD_STORE.SESS]
-
-def get_init_model_and_sess(SD, device_name, gpu_count, segments_per_host,
- model_architecture, compile_params):
- # If a live session is present, re-use it. Otherwise, recreate it.
- if SD_STORE.SESS in SD :
- if SD_STORE.SEGMENT_MODEL not in SD:
- plpy.error("Session and model should exist in SD after the first row"
- "of the first iteration")
- sess = SD[SD_STORE.SESS]
- segment_model = SD[SD_STORE.SEGMENT_MODEL]
- K.set_session(sess)
- else:
- sess = get_keras_session(device_name, gpu_count, segments_per_host)
- K.set_session(sess)
- segment_model = init_model(model_architecture, compile_params)
- SD_STORE.init_SD(SD, sess, segment_model)
- return segment_model, sess
-
-@MinWarning("warning")
-def fit(schema_madlib, source_table, model, model_arch_table,
- model_id, compile_params, fit_params, num_iterations,
- use_gpus, validation_table=None,
- metrics_compute_frequency=None, warm_start=False, name="",
- description="", **kwargs):
-
- module_name = 'madlib_keras_fit'
- fit_params = "" if not fit_params else fit_params
- _assert(compile_params, "Compile parameters cannot be empty or NULL.")
-
- mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
- mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
-
- dep_shape_col = add_postfix(
- MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL, "_shape")
- ind_shape_col = add_postfix(
- MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, "_shape")
-
- segments_per_host = get_segments_per_host()
- use_gpus = use_gpus if use_gpus else False
- if use_gpus:
- accessible_gpus_for_seg = get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name)
- else:
- accessible_gpus_for_seg = get_seg_number()*[0]
-
- fit_validator = FitInputValidator(
- source_table, validation_table, model, model_arch_table,
- model_id, mb_dep_var_col, mb_indep_var_col,
- num_iterations, metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg)
- if metrics_compute_frequency is None:
- metrics_compute_frequency = num_iterations
-
-
- warm_start = bool(warm_start)
-
- # The following two times must be recorded together.
- metrics_elapsed_start_time = time.time()
- start_training_time = datetime.datetime.now()
- #TODO add a unit test for this in a future PR
- # save the original value of the env variable so that we can reset it later.
- original_cuda_env = None
- if CUDA_VISIBLE_DEVICES_KEY in os.environ:
- original_cuda_env = os.environ[CUDA_VISIBLE_DEVICES_KEY]
-
- # Get the serialized master model
- start_deserialization = time.time()
- model_arch, model_weights = get_model_arch_weights(model_arch_table, model_id)
- num_classes = get_num_classes(model_arch)
- input_shape = get_input_shape(model_arch)
- fit_validator.validate_input_shapes(input_shape)
- dist_key_col = '0' if is_platform_pg() else DISTRIBUTION_KEY_COLNAME
- gp_segment_id_col = '0' if is_platform_pg() else GP_SEGMENT_ID_COLNAME
-
- serialized_weights = get_initial_weights(model, model_arch, model_weights,
- warm_start, use_gpus, accessible_gpus_for_seg)
- # Compute total images on each segment
- dist_key_mapping, images_per_seg_train = get_image_count_per_seg_for_minibatched_data_from_db(source_table)
-
-
- if validation_table:
- seg_ids_val, images_per_seg_val = get_image_count_per_seg_for_minibatched_data_from_db(validation_table)
-
- # Construct validation dataset if provided
- validation_set_provided = bool(validation_table)
- validation_metrics = []; validation_loss = []
-
- # Prepare the SQL for running distributed training via UDA
- compile_params_to_pass = "$madlib$" + compile_params + "$madlib$"
- fit_params_to_pass = "$madlib$" + fit_params + "$madlib$"
- run_training_iteration = plpy.prepare("""
- SELECT {schema_madlib}.fit_step(
- {mb_dep_var_col},
- {mb_indep_var_col},
- {dep_shape_col},
- {ind_shape_col},
- $MAD${model_arch}$MAD$::TEXT,
- {compile_params_to_pass}::TEXT,
- {fit_params_to_pass}::TEXT,
- {dist_key_col},
- ARRAY{dist_key_mapping},
- {gp_segment_id_col},
- {segments_per_host},
- ARRAY{images_per_seg_train},
- {use_gpus}::BOOLEAN,
- ARRAY{accessible_gpus_for_seg},
- $1,
- $2
- ) AS iteration_result
- FROM {source_table}
- """.format(**locals()), ["bytea", "boolean"])
-
- # Define the state for the model and loss/metric storage lists
- training_loss, training_metrics, metrics_elapsed_time = [], [], []
- metrics_iters = []
-
- # get the size of serialized model weights string in KB
- model_size = sys.getsizeof(serialized_weights)/1024.0
-
- # Run distributed training for specified number of iterations
- for i in range(1, num_iterations+1):
- start_iteration = time.time()
- is_final_iteration = (i == num_iterations)
- serialized_weights = plpy.execute(run_training_iteration,
- [serialized_weights, is_final_iteration]
- )[0]['iteration_result']
- end_iteration = time.time()
- info_str = "\tTime for training in iteration {0}: {1} sec".format(i,
- end_iteration - start_iteration)
-
- if should_compute_metrics_this_iter(i, metrics_compute_frequency,
- num_iterations):
- # Compute loss/accuracy for training data.
- compute_out = compute_loss_and_metrics(
- schema_madlib, source_table, compile_params_to_pass, model_arch,
- serialized_weights, use_gpus, accessible_gpus_for_seg, dist_key_mapping,
- images_per_seg_train, training_metrics, training_loss, i, is_final_iteration)
- metrics_iters.append(i)
- compute_time, compute_metrics, compute_loss = compute_out
-
- info_str += "\n\tTime for evaluating training dataset in "\
- "iteration {0}: {1} sec\n".format(i, compute_time)
- info_str += "\tTraining set metric after iteration {0}: {1}\n".format(
- i, compute_metrics)
- info_str += "\tTraining set loss after iteration {0}: {1}".format(
- i, compute_loss)
-
- if validation_set_provided:
- # Compute loss/accuracy for validation data.
- val_compute_out = compute_loss_and_metrics(
- schema_madlib, validation_table, compile_params_to_pass,
- model_arch, serialized_weights, use_gpus, accessible_gpus_for_seg,
- seg_ids_val, images_per_seg_val, validation_metrics,
- validation_loss, i, is_final_iteration)
- val_compute_time, val_compute_metrics, val_compute_loss = val_compute_out
-
- info_str += "\n\tTime for evaluating validation dataset in "\
- "iteration {0}: {1} sec\n".format(i, val_compute_time)
- info_str += "\tValidation set metric after iteration {0}: {1}\n".format(
- i, val_compute_metrics)
- info_str += "\tValidation set loss after iteration {0}: {1}".format(
- i, val_compute_loss)
-
- metrics_elapsed_end_time = time.time()
- metrics_elapsed_time.append(
- metrics_elapsed_end_time-metrics_elapsed_start_time)
- plpy.info("\n"+info_str)
- end_training_time = datetime.datetime.now()
-
- version = madlib_version(schema_madlib)
- src_summary_dict = get_source_summary_table_dict(fit_validator)
- class_values = src_summary_dict['class_values']
- class_values_type = src_summary_dict['class_values_type']
- norm_const = src_summary_dict['norm_const']
- norm_const_type = src_summary_dict['norm_const_type']
- dep_vartype = src_summary_dict['dep_vartype']
- dependent_varname = src_summary_dict['dependent_varname_in_source_table']
- independent_varname = src_summary_dict['independent_varname_in_source_table']
- # Define some constants to be inserted into the summary table.
- model_type = "madlib_keras"
- compile_params_dict = convert_string_of_args_to_dict(compile_params)
- metrics_list = get_metrics_from_compile_param(compile_params)
- is_metrics_specified = True if metrics_list else False
- metrics_type = 'ARRAY{0}'.format(metrics_list) if is_metrics_specified else 'NULL'
- metrics_iters = metrics_iters if metrics_iters else 'NULL'
- # We always compute the training loss and metrics, at least once.
- training_loss_final = training_loss[-1]
- training_loss = 'ARRAY{0}'.format(training_loss) if training_loss else 'NULL'
- training_metrics_final, training_metrics = get_metrics_sql_string(
- training_metrics, is_metrics_specified)
- # Validation loss and metrics are computed only if validation_table
- # is provided.
- if validation_set_provided:
- validation_metrics_final, validation_metrics = get_metrics_sql_string(
- validation_metrics, is_metrics_specified)
- validation_loss_final = validation_loss[-1]
- validation_loss = 'ARRAY{0}'.format(validation_loss)
- # Must quote the string before inserting to table. Explicitly
- # quoting it here since this can also take a NULL value, done
- # in the else part.
- validation_table = "$MAD${0}$MAD$".format(validation_table)
- else:
- validation_metrics = validation_loss = 'NULL'
- validation_metrics_final = validation_loss_final = 'NULL'
- validation_table = 'NULL'
-
- if warm_start:
- plpy.execute("DROP TABLE {0}, {1}".format
- (model, fit_validator.output_summary_model_table))
- create_output_summary_table = plpy.prepare("""
- CREATE TABLE {output_summary_model_table} AS
- SELECT
- $MAD${source_table}$MAD$::TEXT AS source_table,
- $MAD${model}$MAD$::TEXT AS model,
- $MAD${dependent_varname}$MAD$::TEXT AS dependent_varname,
- $MAD${independent_varname}$MAD$::TEXT AS independent_varname,
- $MAD${model_arch_table}$MAD$::TEXT AS model_arch_table,
- {model_id}::INTEGER AS {model_id_colname},
- $1 AS compile_params,
- $2 AS fit_params,
- {num_iterations}::INTEGER AS num_iterations,
- {validation_table}::TEXT AS validation_table,
- {metrics_compute_frequency}::INTEGER AS metrics_compute_frequency,
- $3 AS name,
- $4 AS description,
- '{model_type}'::TEXT AS model_type,
- {model_size}::DOUBLE PRECISION AS model_size,
- '{start_training_time}'::TIMESTAMP AS start_training_time,
- '{end_training_time}'::TIMESTAMP AS end_training_time,
- $5 AS metrics_elapsed_time,
- '{version}'::TEXT AS madlib_version,
- {num_classes}::INTEGER AS num_classes,
- $6 AS {class_values_colname},
- $MAD${dep_vartype}$MAD$::TEXT AS {dependent_vartype_colname},
- {norm_const}::{FLOAT32_SQL_TYPE} AS {normalizing_const_colname},
- {metrics_type}::TEXT[] AS metrics_type,
- {training_metrics_final}::DOUBLE PRECISION AS training_metrics_final,
- {training_loss_final}::DOUBLE PRECISION AS training_loss_final,
- {training_metrics}::DOUBLE PRECISION[] AS training_metrics,
- {training_loss}::DOUBLE PRECISION[] AS training_loss,
- {validation_metrics_final}::DOUBLE PRECISION AS validation_metrics_final,
- {validation_loss_final}::DOUBLE PRECISION AS validation_loss_final,
- {validation_metrics}::DOUBLE PRECISION[] AS validation_metrics,
- {validation_loss}::DOUBLE PRECISION[] AS validation_loss,
- ARRAY{metrics_iters}::INTEGER[] AS metrics_iters
- """.format(output_summary_model_table=fit_validator.output_summary_model_table,
- class_values_colname=CLASS_VALUES_COLNAME,
- dependent_vartype_colname=DEPENDENT_VARTYPE_COLNAME,
- normalizing_const_colname=NORMALIZING_CONST_COLNAME,
- FLOAT32_SQL_TYPE = FLOAT32_SQL_TYPE,
- model_id_colname = ModelArchSchema.MODEL_ID,
- **locals()),
- ["TEXT", "TEXT", "TEXT", "TEXT", "DOUBLE PRECISION[]", class_values_type])
- plpy.execute(create_output_summary_table,
- [compile_params, fit_params, name,
- description, metrics_elapsed_time, class_values])
-
- plpy.execute("""
- CREATE TABLE {0}
- (model_weights bytea,
- {1} json)""".format(model, ModelArchSchema.MODEL_ARCH))
- insert_output_table = plpy.prepare("""
- INSERT INTO {0} SELECT model_weights, {1}
- FROM (VALUES($1, $2))t(model_weights, {1})
- """.format(model, ModelArchSchema.MODEL_ARCH), ["bytea", "json"])
- plpy.execute(insert_output_table, [serialized_weights, model_arch])
-
- #TODO add a unit test for this in a future PR
- reset_cuda_env(original_cuda_env)
-
-def get_initial_weights(model_table, model_arch, serialized_weights, warm_start,
- use_gpus, accessible_gpus_for_seg, mst_filter=''):
- """
- If warm_start is True, return back initial weights from model table.
- If warm_start is False, first try to get the weights from model_arch
- table, if no weights are defined there, randomly initialize it using
- keras.
- We also need to set the cuda environment variable based on the platform.
- 1. For postgres, if user specifies use_gpus=False which means they want
- to use CPU, then we have to set CUDA_VISIBLE_DEVICES to -1 to disable gpu.
- Otherwise model.get_weights() will use gpu if available.
-
- 2. For gpdb, we want to disable gpu on gpdb's master node because GPUs
- will only be used for segment nodes.
- @args:
- @param model_table: Output model table passed in to fit.
- @param model_arch_result: Dict containing model architecture info.
- @param warm_start: Boolean flag indicating warm start or not.
- """
- if is_platform_pg():
- _ = get_device_name_and_set_cuda_env(accessible_gpus_for_seg[0], None)
- else:
- _ = get_device_name_and_set_cuda_env(0, None)
-
- if warm_start:
- serialized_weights = plpy.execute("""
- SELECT model_weights FROM {model_table} {mst_filter} LIMIT 1
- """.format(**locals()))[0]['model_weights']
- else:
- if not serialized_weights:
- model = model_from_json(model_arch)
- serialized_weights = madlib_keras_serializer.serialize_nd_weights(
- model.get_weights())
- return serialized_weights
-
-def get_source_summary_table_dict(fit_validator):
- source_summary = plpy.execute("""
- SELECT
- {class_values} AS class_values,
- {norm_const} AS norm_const,
- {dep_vartype} AS dep_vartype,
- {dep_varname} AS dependent_varname_in_source_table,
- {indep_varname} AS independent_varname_in_source_table
- FROM {tbl}
- """.format(class_values=CLASS_VALUES_COLNAME,
- norm_const=NORMALIZING_CONST_COLNAME,
- dep_vartype=DEPENDENT_VARTYPE_COLNAME,
- dep_varname='dependent_varname',
- indep_varname='independent_varname',
- tbl=fit_validator.source_summary_table))[0]
- source_summary['class_values_type'] = get_expr_type(
- CLASS_VALUES_COLNAME, fit_validator.source_summary_table)
- source_summary['norm_const_type'] = get_expr_type(
- NORMALIZING_CONST_COLNAME, fit_validator.source_summary_table)
- return source_summary
-
-def get_metrics_sql_string(metrics_list, is_metrics_specified):
- """
- Return the SQL string to use for creating metrics SQL values.
- """
- if is_metrics_specified:
- metrics_final = metrics_list[-1]
- metrics_all = 'ARRAY{0}'.format(metrics_list)
- else:
- metrics_final = metrics_all = 'NULL'
- return metrics_final, metrics_all
-
-def compute_loss_and_metrics(schema_madlib, table, compile_params, model_arch,
- serialized_weights, use_gpus, accessible_gpus_for_seg,
- dist_key_mapping, images_per_seg_val, metrics_list, loss_list,
- curr_iter, is_final_iteration):
- """
- Compute the loss and metric using a given model (serialized_weights) on the
- given dataset (table.)
- """
- start_val = time.time()
- evaluate_result = get_loss_metric_from_keras_eval(schema_madlib,
- table,
- compile_params,
- model_arch,
- serialized_weights,
- use_gpus,
- accessible_gpus_for_seg,
- dist_key_mapping,
- images_per_seg_val,
- is_final_iteration)
- end_val = time.time()
-
- if len(evaluate_result) not in [1, 2]:
- plpy.error('Calling evaluate on table {0} returned < 2 '
- 'metrics. Expected both loss and a metric.'.format(table))
- loss = evaluate_result[0]
- metric = evaluate_result[1]
- metrics_list.append(metric)
- loss_list.append(loss)
- return end_val - start_val, metric, loss
-
-def should_compute_metrics_this_iter(curr_iter, metrics_compute_frequency,
- num_iterations):
- """
- Check if we want to compute loss/accuracy for the current iteration
- :param curr_iter:
- :param metrics_compute_frequency:
- :param num_iterations:
- :return: Returns a boolean
- return TRUE, if it is the last iteration, or if metrics_compute_frequency
- iterations have elapsed since the last time it was computed.
- return FALSE otherwise.
- """
- # Compute loss/accuracy every metrics_compute_frequency'th iteration,
- # and also for the last iteration.
- return (curr_iter)%metrics_compute_frequency == 0 or \
- curr_iter == num_iterations
-
-def init_model(model_architecture, compile_params):
- """
- Should only be called at the first row of first iteration.
- """
- segment_model = model_from_json(model_architecture)
- compile_model(segment_model, compile_params)
- return segment_model
-
-def update_model(segment_model, prev_serialized_weights):
- """
- Happens at first row of each iteration.
- """
- model_shapes = get_model_shapes(segment_model)
- model_weights = madlib_keras_serializer.deserialize_as_nd_weights(
- prev_serialized_weights, model_shapes)
- segment_model.set_weights(model_weights)
-
-def fit_transition(state, dependent_var, independent_var, dependent_var_shape,
- independent_var_shape, model_architecture,
- compile_params, fit_params, dist_key, dist_key_mapping,
- current_seg_id, segments_per_host, images_per_seg, use_gpus,
- accessible_gpus_for_seg, prev_serialized_weights, is_final_iteration=True,
- is_multiple_model=False, **kwargs):
- """
- This transition function is common for madlib_keras_fit() and
- madlib_keras_fit_multiple_model(). The important difference between
- these two calls is the way this function handles the input param
- prev_serialized_weights and clearing keras session.
- For madlib_keras_fit_multiple_model,
- a. prev_serialized_weights is always passed in as the state
- (image count, serialized weights), since it is fetched in the
- table for each hop of the model between segments.
- b. keras session is cleared at the end of each iteration, i.e,
- last row of each iteration.
- For madlib_keras_fit,
- a. prev_serialized_weights is passed in as serialized weights
- b. keras session is cleared at the end of the final iteration,
- i.e, last row of last iteration.
- """
- if not independent_var or not dependent_var:
- return state
- SD = kwargs['SD']
- device_name = get_device_name_and_set_cuda_env(accessible_gpus_for_seg[current_seg_id], current_seg_id)
-
- segment_model, sess = get_init_model_and_sess(SD, device_name,
- accessible_gpus_for_seg[current_seg_id],
- segments_per_host,
- model_architecture, compile_params)
- if not state:
- agg_image_count = 0
- set_model_weights(segment_model, prev_serialized_weights)
- else:
- agg_image_count = float(state)
-
- # Prepare the data
- x_train = np_array_float32(independent_var, independent_var_shape)
- y_train = np_array_int16(dependent_var, dependent_var_shape)
-
- # Fit segment model on data
- #TODO consider not doing this every time
- fit_params = parse_and_validate_fit_params(fit_params)
- segment_model.fit(x_train, y_train, **fit_params)
-
- # Aggregating number of images, loss and accuracy
- agg_image_count += len(x_train)
- total_images = get_image_count_per_seg_from_array(dist_key_mapping.index(dist_key),
- images_per_seg)
- is_last_row = agg_image_count == total_images
- return_state = get_state_to_return(segment_model, is_last_row, is_multiple_model,
- agg_image_count, total_images)
- if is_last_row:
- if is_final_iteration or is_multiple_model:
- SD_STORE.clear_SD(SD)
- clear_keras_session(sess)
-
- return return_state
-
-def get_state_to_return(segment_model, is_last_row, is_multiple_model, agg_image_count,
- total_images):
- """
- 1. For both model averaging fit_transition and fit multiple transition, the
- state only needs to have the image count except for the last row.
- 2. For model averaging fit_transition, the last row state must always contain
- the image count as well as the model weights
- 3. For fit multiple transition, the last row state only needs the model
- weights. This state is the output of the UDA for that hop. We don't need
- the image_count here because unlike model averaging, model hopper does
- not have a merge/final function and there is no need to average the weights
- based on the image count.
- :param segment_model: cached model for that segment
- :param is_last_row: boolean to indicate if last row for that hop
- :param is_multiple_model: boolean
- :param agg_image_count: aggregated image count per hop
- :param total_images: total images per segment
- :return:
- """
- if is_last_row:
- updated_model_weights = segment_model.get_weights()
- if is_multiple_model:
- new_state = madlib_keras_serializer.serialize_nd_weights(updated_model_weights)
- else:
- updated_model_weights = [total_images * w for w in updated_model_weights]
- new_state = madlib_keras_serializer.serialize_state_with_nd_weights(
- agg_image_count, updated_model_weights)
- else:
- new_state = float(agg_image_count)
-
- return new_state
-
-def fit_merge(state1, state2, **kwargs):
-
- # Return if called early
- if not state1 or not state2:
- return state1 or state2
-
- # Deserialize states
- image_count1, weights1 = madlib_keras_serializer.deserialize_as_image_1d_weights(state1)
- image_count2, weights2 = madlib_keras_serializer.deserialize_as_image_1d_weights(state2)
-
- # Compute total image counts
- image_count = (image_count1 + image_count2) * 1.0
-
- # Aggregate the weights
- total_weights = weights1 + weights2
-
- # Return the merged state
- return madlib_keras_serializer.serialize_state_with_1d_weights(
- image_count, total_weights)
-
-def fit_final(state, **kwargs):
- # Return if called early
- if not state:
- return state
-
- image_count, weights = madlib_keras_serializer.deserialize_as_image_1d_weights(state)
- if image_count == 0:
- plpy.error("fit_final: Total images processed is 0")
-
- # Averaging the weights
- weights /= image_count
- return madlib_keras_serializer.serialize_nd_weights(weights)
-
-
-def evaluate(schema_madlib, model_table, test_table, output_table,
- use_gpus, mst_key, **kwargs):
-
- module_name = 'madlib_keras_evaluate'
- is_mult_model = mst_key is not None
- if test_table:
- test_summary_table = add_postfix(test_table, "_summary")
- model_summary_table = None
- if model_table:
- model_summary_table = add_postfix(model_table, "_summary")
-
- mult_where_clause = ""
- if is_mult_model:
- mult_where_clause = "WHERE mst_key = {0}".format(mst_key)
- model_summary_table = create_summary_view(module_name, model_table, mst_key)
-
- validate_evaluate(module_name, model_table, model_summary_table, test_table, test_summary_table, output_table, is_mult_model)
-
- segments_per_host = get_segments_per_host()
- if use_gpus:
- accessible_gpus_for_seg = get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name)
- else:
- accessible_gpus_for_seg = get_seg_number()*[0]
-
- model_weights_query = "SELECT model_weights, model_arch FROM {0} {1}".format(
- model_table, mult_where_clause)
-
- res = plpy.execute(model_weights_query)[0]
- _assert(res, "{0}: The model does not exist.")
- model_weights = res['model_weights']
- model_arch = res['model_arch']
-
- input_shape = get_input_shape(model_arch)
- InputValidator.validate_input_shape(
- test_table, MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, input_shape, 2, True)
-
- compile_params_query = "SELECT compile_params, metrics_type FROM {0}".format(model_summary_table)
- res = plpy.execute(compile_params_query)[0]
- metrics_type = res['metrics_type']
- compile_params = "$madlib$" + res['compile_params'] + "$madlib$"
-
- dist_key_mapping, images_per_seg = get_image_count_per_seg_for_minibatched_data_from_db(test_table)
-
- loss, metric = \
- get_loss_metric_from_keras_eval(
- schema_madlib, test_table, compile_params, model_arch,
- model_weights, use_gpus, accessible_gpus_for_seg, dist_key_mapping,
- images_per_seg)
-
- if not metrics_type:
- metrics_type = None
- metric = None
-
- with MinWarning("error"):
- create_output_table = plpy.prepare("""
- CREATE TABLE {0} AS
- SELECT $1 as loss, $2 as metric, $3 as metrics_type""".format(output_table), ["FLOAT", "FLOAT", "TEXT[]"])
- plpy.execute(create_output_table, [loss, metric, metrics_type])
-
- if is_mult_model:
- plpy.execute("DROP VIEW IF EXISTS {0}".format(model_summary_table))
-
-def validate_evaluate(module_name, model_table, model_summary_table, test_table, test_summary_table, output_table, is_mult_model):
- def _validate_test_summary_tbl():
- input_tbl_valid(test_summary_table, module_name,
- error_suffix_str="Please ensure that the test table ({0}) "
- "has been preprocessed by "
- "the image preprocessor.".format(test_table))
- cols_in_tbl_valid(test_summary_table, [CLASS_VALUES_COLNAME,
- NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME,
- DEPENDENT_VARNAME_COLNAME, INDEPENDENT_VARNAME_COLNAME], module_name)
-
- input_tbl_valid(model_table, module_name)
- if is_mult_model and not columns_exist_in_table(model_table, ['mst_key']):
- plpy.error("{module_name}: Single model should not pass mst_key".format(**locals()))
- if not is_mult_model and columns_exist_in_table(model_table, ['mst_key']):
- plpy.error("{module_name}: Multi-model needs to pass mst_key".format(**locals()))
- InputValidator.validate_predict_evaluate_tables(
- module_name, model_table, model_summary_table,
- test_table, output_table, MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL)
- _validate_test_summary_tbl()
- validate_bytea_var_for_minibatch(test_table,
- MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL)
-
-def get_loss_metric_from_keras_eval(schema_madlib, table, compile_params,
- model_arch, serialized_weights, use_gpus,
- accessible_gpus_for_seg, dist_key_mapping, images_per_seg,
- is_final_iteration=True):
-
- dist_key_col = '0' if is_platform_pg() else DISTRIBUTION_KEY_COLNAME
- gp_segment_id_col = '0' if is_platform_pg() else GP_SEGMENT_ID_COLNAME
- segments_per_host = get_segments_per_host()
-
- mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
- mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
-
- dep_shape_col = add_postfix(
- MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL, "_shape")
- ind_shape_col = add_postfix(
- MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, "_shape")
- """
- This function will call the internal keras evaluate function to get the loss
- and accuracy of each tuple which then gets averaged to get the final result.
- """
- use_gpus = use_gpus if use_gpus else False
- evaluate_query = plpy.prepare("""
- select ({schema_madlib}.internal_keras_evaluate(
- {mb_dep_var_col},
- {mb_indep_var_col},
- {dep_shape_col},
- {ind_shape_col},
- $MAD${model_arch}$MAD$,
- $1,
- {compile_params},
- {dist_key_col},
- ARRAY{dist_key_mapping},
- {gp_segment_id_col},
- {segments_per_host},
- ARRAY{images_per_seg},
- {use_gpus}::BOOLEAN,
- ARRAY{accessible_gpus_for_seg},
- {is_final_iteration}
- )) as loss_metric
- from {table}
- """.format(**locals()), ["bytea"])
- res = plpy.execute(evaluate_query, [serialized_weights])
- loss_metric = res[0]['loss_metric']
- return loss_metric
-
-
-def internal_keras_eval_transition(state, dependent_var, independent_var,
- dependent_var_shape, independent_var_shape,
- model_architecture, serialized_weights, compile_params,
- dist_key, dist_key_mapping, current_seg_id,
- segments_per_host, images_per_seg,
- use_gpus, accessible_gpus_for_seg,
- is_final_iteration, **kwargs):
- SD = kwargs['SD']
- device_name = get_device_name_and_set_cuda_env(accessible_gpus_for_seg[current_seg_id], current_seg_id)
- agg_loss, agg_metric, agg_image_count = state
-
- # This transition function is common to evaluate as well as the fit functions
- # and is used to determine when to clear the session.
- # For evaluate,
- # is_final_iteration is always set to true, so the session is cleared once
- # evaluated the last buffer on each segment.
- # When called from fit functions,
- # if is_final_iteration is false, the fit function has already created a
- # session and a graph that can be used between iterations and cleared only
- # for the last buffer of last iteration
- # if is_final_iteration is false, we can clear the
-
- segment_model, sess = get_init_model_and_sess(SD, device_name,
- accessible_gpus_for_seg[current_seg_id],
- segments_per_host,
- model_architecture,
- compile_params)
- if not agg_image_count:
- # These should already be 0, but just in case make sure
- agg_metric = 0
- agg_loss = 0
- set_model_weights(segment_model, serialized_weights)
-
- x_val = np_array_float32(independent_var, independent_var_shape)
- y_val = np_array_int16(dependent_var, dependent_var_shape)
-
- with K.tf.device(device_name):
- res = segment_model.evaluate(x_val, y_val)
-
- # if metric is None, model.evaluate will only return loss as a scalar
- # Otherwise, it will return a list which has loss and metric
- if type(res) is list:
- loss, metric = res
- else:
- loss = res
- metric = 0
-
- image_count = len(y_val)
-
- agg_image_count += image_count
- agg_loss += (image_count * loss)
- agg_metric += (image_count * metric)
-
- total_images = get_image_count_per_seg_from_array(dist_key_mapping.index(dist_key),
- images_per_seg)
-
- if agg_image_count == total_images and is_final_iteration:
- K.clear_session()
- sess.close()
- SD_STORE.clear_SD(SD)
- del segment_model
- del sess
-
- state[0] = agg_loss
- state[1] = agg_metric
- state[2] = agg_image_count
-
- return state
-
-def internal_keras_eval_merge(state1, state2, **kwargs):
- # If either state is None, return the other one
- if not state1 or not state2:
- return state1 or state2
-
- loss1, metric1, image_count1 = state1
- loss2, metric2, image_count2 = state2
-
- merged_loss = loss1 + loss2
- merged_metric = metric1 + metric2
- total_image_count = image_count1 + image_count2
-
- merged_state = [ merged_loss, merged_metric , total_image_count ]
-
- return merged_state
-
-def internal_keras_eval_final(state, **kwargs):
- loss, metric, image_count = state
-
- if image_count == 0:
- plpy.error("internal_keras_eval_final: Total images processed is 0")
-
- loss /= image_count
- metric /= image_count
-
- return loss, metric
-
-def fit_help(schema_madlib, message, **kwargs):
- """
- Help function for keras fit
-
- Args:
- @param schema_madlib
- @param message: string, Help message string
- @param kwargs
-
- Returns:
- String. Help/usage information
- """
- if not message:
- help_string = """
------------------------------------------------------------------------
- SUMMARY
------------------------------------------------------------------------
-This module allows you to use SQL to call deep learning
-models designed in Keras, which is a high-level neural
-network API written in Python.
-Keras was developed for fast experimentation. It can run
-on top of different backends and the one that is currently
-supported by MADlib is TensorFlow. The implementation
-in MADlib is distributed and designed to train
-a single large model across multiple segments (workers)
-in a Greenplum database. PostgreSQL is also supported.
-
-For more details on function usage:
- SELECT {schema_madlib}.madlib_keras_fit('usage')
- """
- elif message in ['usage', 'help', '?']:
- help_string = """
------------------------------------------------------------------------
- USAGE
------------------------------------------------------------------------
- SELECT {schema_madlib}.madlib_keras_fit(
- source_table, -- Name of the table containing the
- training data
- model, -- Name of the output table containing
- the model
- model_arch_table, -- Name of the table containing the
- model architecture
- model_id, -- This is the id in 'model_arch_table'
- containing the model architecture
- compile_params, -- Parameters passed to the compile
- method of the Keras model class
- fit_params, -- Parameters passed to the fit method
- of the Keras model class
- num_iterations, -- Number of iterations to train.
- use_gpus, -- Flag to enable GPU support
- validation_table, -- Name of the table containing
- the validation dataset
- metrics_compute_frequency, -- Frequency to compute per-iteration
- metrics
- warm_start, -- Flag to enable warm start
- name, -- Free text string to identify a name
- description -- Free text string to provide a description
- )
- );
-
------------------------------------------------------------------------
- OUTPUT
------------------------------------------------------------------------
-The output table ('model' above) contains the following columns:
-
-model_weights: Byte array containing the weights of the neural net.
-model_arch: A JSON representation of the model architecture used in
- training.
-
-A summary table ('<model>_summary') is created to store various training
-statistics as well as the input parameters.
-"""
- else:
- help_string = "No such option. Use {schema_madlib}.madlib_keras_fit()"
-
- return help_string.format(schema_madlib=schema_madlib)
-# ---------------------------------------------------------------------
-
-
-def evaluate_help(schema_madlib, message, **kwargs):
- """
- Help function for keras evaluate
-
- Args:
- @param schema_madlib
- @param message: string, Help message string
- @param kwargs
-
- Returns:
- String. Help/usage information
- """
- if not message:
- help_string = """
------------------------------------------------------------------------
- SUMMARY
------------------------------------------------------------------------
-This function allows the user to evaluate a madlib_keras_fit trained
-model.
-
-For more details on function usage:
- SELECT {schema_madlib}.madlib_keras_evaluate('usage')
- """
- elif message in ['usage', 'help', '?']:
- help_string = """
------------------------------------------------------------------------
- USAGE
------------------------------------------------------------------------
- SELECT {schema_madlib}.madlib_keras_evaluate(
- model_table, -- Name of the table containing the model
- test_table, -- Name of the table containing the evaluation dataset
- output_table, -- Name of the output table
- use_gpus, -- Flag to enable GPU support
- mst_key -- Identifier for the desired model out of multimodel
- training output
- )
- );
-
------------------------------------------------------------------------
- OUTPUT
------------------------------------------------------------------------
-The output table ('output_table' above) contains the following columns:
-
-loss: Loss value on evaluation dataset.
-metric: Metric value on evaluation dataset, where 'metrics_type'
- below identifies the type of metric.
-metrics_type: Type of metric used that was used in the training step.
-"""
- else:
- help_string = "No such option. Use {schema_madlib}.madlib_keras_evaluate()"
-
- return help_string.format(schema_madlib=schema_madlib)
-# ---------------------------------------------------------------------
-
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
deleted file mode 100644
index 90e7a98..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
+++ /dev/null
@@ -1,2162 +0,0 @@
-/* ----------------------------------------------------------------------- *//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *
- * @file madlib_keras.sql_in
- *
- * @brief SQL functions for distributed deep learning with keras
- * @date June 2019
- *
- *
- *//* ----------------------------------------------------------------------- */
-
-m4_include(`SQLCommon.m4')
-
-/**
-@addtogroup grp_keras
-
-@brief Fit, evaluate and predict using the Keras API.
-
-<div class="toc"><b>Contents</b><ul>
-<li class="level1"><a href="#keras_fit">Fit</a></li>
-<li class="level1"><a href="#keras_evaluate">Evaluate</a></li>
-<li class="level1"><a href="#keras_predict">Predict</a></li>
-<li class="level1"><a href="#keras_predict_byom">Predict BYOM</a></li>
-<li class="level1"><a href="#example">Examples</a></li>
-<li class="level1"><a href="#notes">Notes</a></li>
-<li class="level1"><a href="#background">Technical Background</a></li>
-<li class="level1"><a href="#literature">Literature</a></li>
-<li class="level1"><a href="#related">Related Topics</a></li>
-</ul></div>
-
-\warning <em> This MADlib method is still in early stage development.
-Interface and implementation are subject to change. </em>
-
-This module allows you to use SQL to call deep learning
-models designed in Keras [1], which is a high-level neural
-network API written in Python.
-Keras was developed for fast experimentation. It can run
-on top of different backends and the one that is currently
-supported by MADlib is TensorFlow [2]. The implementation
-in MADlib is distributed and designed to train
-a single model across multiple segments (workers)
-in Greenplum database. (PostgreSQL is also supported.)
-Alternatively, to train multiple models at the same time for model
-architecture search or hyperparameter tuning, you can
-use <a href="group__grp__keras__run__model__selection.html">Model Selection</a>.
-
-The main use case is image classification
-using sequential models, which are made up of a
-linear stack of layers. This includes multilayer perceptrons (MLPs)
-and convolutional neural networks (CNNs). Regression is not
-currently supported.
-
-Before using Keras in MADlib you will need to mini-batch
-your training and evaluation datasets by calling the
-<a href="group__grp__input__preprocessor__dl.html">Preprocessor
-for Images</a> which is a utility that prepares image data for
-use by models that support mini-batch as an optimization option.
-This is a one-time operation and you would only
-need to re-run the preprocessor if your input data has changed.
-The advantage of using mini-batching is that it
-can perform better than stochastic gradient descent
-because it uses more than one training example at a time,
-typically resulting faster and smoother convergence [3].
-
-You can also do inference on models that have not been trained in MADlib,
-but rather imported from an external source. This is in the section
-called "Predict BYOM" below, where "BYOM" stands for "Bring Your Own Model."
-
-Note that the following MADlib functions are targeting a specific Keras
-version (2.2.4) with a specific TensorFlow kernel version (1.14).
-Using a newer or older version may or may not work as intended.
-
-@note CUDA GPU memory cannot be released until the process holding it is terminated.
-When a MADlib deep learning function is called with GPUs, Greenplum internally
-creates a process (called a slice) which calls TensorFlow to do the computation.
-This process holds the GPU memory until one of the following two things happen:
-query finishes and user logs out of the Postgres client/session; or,
-query finishes and user waits for the timeout set by gp_vmem_idle_resource_timeout.
-The default value for this timeout is 18 sec [8]. So the recommendation is:
-log out/reconnect to the session after every GPU query; or
-wait for gp_vmem_idle_resource_timeout before you run another GPU query (you can
-also set it to a lower value).
-
-@anchor keras_fit
-@par Fit
-The fit (training) function has the following format:
-
-<pre class="syntax">
-madlib_keras_fit(
- source_table,
- model,
- model_arch_table,
- model_id,
- compile_params,
- fit_params,
- num_iterations,
- use_gpus,
- validation_table,
- metrics_compute_frequency,
- warm_start,
- name,
- description
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
- <dt>source_table</dt>
- <dd>TEXT. Name of the table containing the training data.
- This is the name of the output
- table from the image preprocessor. Independent
- and dependent variables are specified in the preprocessor
- step which is why you do not need to explictly state
- them here as part of the fit function.</dd>
-
- <dt>model</dt>
- <dd>TEXT. Name of the output table containing the model.
- Details of the output table are shown below.
- </dd>
-
- <dt>model_arch_table</dt>
- <dd>TEXT. Name of the table containing the model
- architecture and (optionally) initial weights to use for
- training.
- </dd>
-
- <dt>model_id</dt>
- <dd>INTEGER. This is the id in 'model_arch_table'
- containing the model architecture and (optionally)
- initial weights to use for training.
- </dd>
-
- <DT>compile_params</DT>
- <DD>TEXT.
- Parameters passed to the compile method of the Keras
- model class [4]. These parameters will be passed through as is
- so they must conform to the Keras API definition.
- As an example, you might use something like: <em>loss='categorical_crossentropy', optimizer='adam', metrics=['acc']</em>.
- The mandatory parameters that must be specified are 'optimizer'
- and 'loss'. Others are optional and will use the default
- values as per Keras if not specified here. Also, when
- specifying 'loss' and 'metrics' do <em>not</em> include the
- module and submodule prefixes
- like <em>loss='losses.categorical_crossentropy'</em>
- or <em>optimizer='keras.optmizers.adam'</em>.
-
- @note
- The following loss function is
- not supported: <em>sparse_categorical_crossentropy</em>.
- The following metrics are not
- supported: <em>sparse_categorical_accuracy, top_k_categorical_accuracy, sparse_top_k_categorical_accuracy</em> and custom metrics.
-
- </DD>
-
- <DT>fit_params </DT>
- <DD>TEXT. Parameters passed to the fit method of the Keras
- model class [4]. These will be passed through as is
- so they must conform to the Keras API definition.
- As an example, you might use something like:
- <em>batch_size=128, epochs=4</em>.
- There are no mandatory parameters so
- if you specify NULL, it will use all default
- values as per Keras.
- </DD>
-
- <DT>num_iterations</DT>
- <DD>INTEGER. Number of iterations to train.
- </DD>
-
- <DT>use_gpus (optional)</DT>
- <DD>BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs
- are to be used for training the neural network. Set to TRUE to use GPUs.
-
- @note
- This parameter must not conflict with how the distribution rules are set in
- the preprocessor function. For example, if you set a distribution rule to use
- certain segments on hosts that do not have GPUs attached, you will get an error
- if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues
- when segments share GPU resources.
- For example, if you have 1 GPU per segment host and your cluster has 4
- segments per segment host, it means that all 4
- segments will share the same
- GPU on each host. The current recommended
- configuration is 1 GPU per segment.
- </DD>
-
- <dt>validation_table (optional)</dt>
- <dd>TEXT, default: none. Name of the table containing
- the validation dataset.
- Note that the validation dataset must be preprocessed
- in the same way as the training dataset, so this
- is the name of the output
- table from running the image preprocessor on the validation dataset.
- Using a validation dataset can mean a
- longer training time, depending on its size.
- This can be controlled using the 'metrics_compute_frequency'
- paremeter described below.</dd>
-
- <DT>metrics_compute_frequency (optional)</DT>
- <DD>INTEGER, default: once at the end of training
- after 'num_iterations'. Frequency to compute per-iteration
- metrics for the training dataset and validation dataset
- (if specified). There can be considerable cost to
- computing metrics every iteration, especially if the
- training dataset is large. This parameter is a way of
- controlling the frequency of those computations.
- For example, if you specify 5, then metrics will be computed
- every 5 iterations as well as at the end of training
- after 'num_iterations'. If you use the default,
- metrics will be computed only
- once after 'num_iterations' have completed.
- </DD>
-
- <DT>warm_start (optional)</DT>
- <DD>BOOLEAN, default: FALSE.
- Initalize weights with the coefficients
- from the last call of the fit
- function. If set to TRUE, weights will be
- initialized from the model table
- generated by the previous training run.
-
- @note
- The warm start feature works based on the name of the
- model output table from a previous training run.
- When using warm start, do not drop the model output table
- or the model output summary table
- before calling the fit function, since these are needed to obtain the
- weights from the previous run.
- If you are not using warm start, the model output table
- and the model output table summary must be dropped in
- the usual way before calling the training function.
- </DD>
-
- <DT>name (optional)</DT>
- <DD>TEXT, default: NULL.
- Free text string to identify a name, if desired.
- </DD>
-
- <DT>description (optional)</DT>
- <DD>TEXT, default: NULL.
- Free text string to provide a description, if desired.
- </DD>
-</dl>
-
-<b>Output tables</b>
-<br>
- The model table produced by fit contains the following columns:
- <table class="output">
- <tr>
- <th>model_weights</th>
- <td>BYTEA8. Byte array containing the weights of the neural net.</td>
- </tr>
- <tr>
- <th>model_arch</th>
- <td>TEXT. A JSON representation of the model architecture
- used in training.</td>
- </tr>
- </table>
-
-A summary table named \<model\>_summary is also created, which has the following columns:
- <table class="output">
- <tr>
- <th>source_table</th>
- <td>Source table used for training.</td>
- </tr>
- <tr>
- <th>model</th>
- <td>Model output table produced by training.</td>
- </tr>
- <tr>
- <th>independent_varname</th>
- <td>Independent variables column from the original
- source table in the image preprocessing step.</td>
- </tr>
- <tr>
- <th>dependent_varname</th>
- <td>Dependent variable column from the original
- source table in the image preprocessing step.</td>
- </tr>
- <tr>
- <th>model_arch_table</th>
- <td>Name of the table containing
- the model architecture and (optionally) the
- initial model weights.</td>
- </tr>
- <tr>
- <th>model_id</th>
- <td>The id of the model in
- the model architecture table used for training.</td>
- </tr>
- <tr>
- <th>compile_params</th>
- <td>Compile parameters passed to Keras.</td>
- </tr>
- <tr>
- <th>fit_params</th>
- <td>Fit parameters passed to Keras.</td>
- </tr>
- <tr>
- <th>num_iterations</th>
- <td>Number of iterations of training completed.</td>
- </tr>
- <tr>
- <th>validation_table</th>
- <td>Name of the table containing
- the validation dataset (if specified).</td>
- </tr>
- <tr>
- <th>metrics_compute_frequency</th>
- <td>Frequency that per-iteration metrics are computed
- for the training dataset and validation
- dataset.</td>
- </tr>
- <tr>
- <th>name</th>
- <td>Name of the training run (free text).</td>
- </tr>
- <tr>
- <th>description</th>
- <td>Description of the training run (free text).</td>
- </tr>
- <tr>
- <th>model_type</th>
- <td>General identifier for type of model trained.
- Currently says 'madlib_keras'.</td>
- </tr>
- <tr>
- <th>model_size</th>
- <td>Size of the model in KB. Models are stored in
- 'bytea' data format which is used for binary strings
- in PostgreSQL type databases.</td>
- </tr>
- <tr>
- <th>start_training_time</th>
- <td>Timestamp for start of training.</td>
- </tr>
- <tr>
- <th>end_training_time</th>
- <td>Timestamp for end of training.</td>
- </tr>
- <tr>
- <th>metrics_elapsed_time</th>
- <td> Array of elapsed time for metric computations as
- per the 'metrics_compute_frequency' parameter.
- Useful for drawing a curve showing loss, accuracy or
- other metrics as a function of time.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of elapsed time for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>madlib_version</th>
- <td>Version of MADlib used.</td>
- </tr>
- <tr>
- <th>num_classes</th>
- <td>Count of distinct classes values used.</td>
- </tr>
- <tr>
- <th>class_values</th>
- <td>Array of actual class values used.</td>
- </tr>
- <tr>
- <th>dependent_vartype</th>
- <td>Data type of the dependent variable.</td>
- </tr>
- <tr>
- <th>normalizing_constant</th>
- <td>Normalizing constant used from the
- image preprocessing step.</td>
- </tr>
- <tr>
- <th>metrics_type</th>
- <td>Metric specified in the 'compile_params'.</td>
- </tr>
- <tr>
- <th>training_metrics_final</th>
- <td>Final value of the training
- metric after all iterations have completed.
- The metric reported is the one
- specified in the 'metrics_type' parameter.</td>
- </tr>
- <tr>
- <th>training_loss_final</th>
- <td>Final value of the training loss after all
- iterations have completed.</td>
- </tr>
- <tr>
- <th>training_metrics</th>
- <td>Array of training metrics as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of metrics for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>training_loss</th>
- <td>Array of training losses as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of losses for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>validation_metrics_final</th>
- <td>Final value of the validation
- metric after all iterations have completed.
- The metric reported is the one
- specified in the 'metrics_type' parameter.</td>
- </tr>
- <tr>
- <th>validation_loss_final</th>
- <td>Final value of the validation loss after all
- iterations have completed.</td>
- </tr>
- <tr>
- <th>validation_metrics</th>
- <td>Array of validation metrics as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of metrics for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>validation_loss</th>
- <td>Array of validation losses as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of losses for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>metrics_iters</th>
- <td>Array indicating the iterations for which
- metrics are calculated, as derived from the
- parameters 'num_iterations' and 'metrics_compute_frequency'.
- For example, if 'num_iterations=5'
- and 'metrics_compute_frequency=2', then 'metrics_iters' value
- would be {2,4,5} indicating that metrics were computed
- at iterations 2, 4 and 5 (at the end).
- If 'num_iterations=5'
- and 'metrics_compute_frequency=1', then 'metrics_iters' value
- would be {1,2,3,4,5} indicating that metrics were computed
- at every iteration.</td>
- </tr>
- </table>
-
-@anchor keras_evaluate
-@par Evaluate
-The evaluation function has the following format:
-
-<pre class="syntax">
-madlib_keras_evaluate(
- model_table,
- test_table,
- output_table,
- use_gpus,
- mst_key
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
-
-<DT>model_table</DT>
- <DD>TEXT. Name of the table containing the model
- to use for validation.
- </DD>
-
- <DT>test_table</DT>
- <dd>TEXT. Name of the table containing the evaluation dataset.
- Note that test/validation data must be preprocessed in the same
- way as the training dataset, so
- this is the name of the output
- table from the image preprocessor. Independent
- and dependent variables are specified in the preprocessor
- step which is why you do not need to explictly state
- them here as part of the fit function.</dd>
-
- <DT>output_table</DT>
- <DD>TEXT. Name of table that validation output will be
- written to. Table contains:</DD>
- <table class="output">
- <tr>
- <th>loss</th>
- <td>Loss value on evaluation dataset.</td>
- </tr>
- <tr>
- <th>metric</th>
- <td>Metric value on evaluation dataset, where 'metrics_type'
- below identifies the type of metric.</td>
- </tr>
- <tr>
- <th>metrics_type</th>
- <td>Type of metric used that was used in the training step.</td>
- </tr>
-
- <DT>use_gpus (optional)</DT>
- <DD>BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs
- are to be used for training the neural network. Set to TRUE to use GPUs.
-
- @note
- This parameter must not conflict with how the distribution rules are set in
- the preprocessor function. For example, if you set a distribution rule to use
- certain segments on hosts that do not have GPUs attached, you will get an error
- if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues
- when segments share GPU resources.
- For example, if you have 1 GPU per segment host and your cluster has 4
- segments per segment host, it means that all 4
- segments will share the same
- GPU on each host. The current recommended
- configuration is 1 GPU per segment.
- </DD>
-
- <DT>mst_key (optional)</DT>
- <DD>INTEGER, default: NULL. ID that defines a unique tuple for
- model architecture-compile parameters-fit parameters in a model
- selection table. Do not use this if training one model at a time using madlib_keras_fit().
- See the <a href="group__grp__keras__run__model__selection.html">Model Selection</a> section
- for more details on model selection by training multiple models at a time.
- </DD>
-</DL>
-
-@anchor keras_predict
-@par Predict
-The prediction function has the following format:
-<pre class="syntax">
-madlib_keras_predict(
- model_table,
- test_table,
- id_col,
- independent_varname,
- output_table,
- pred_type,
- use_gpus,
- mst_key
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
-
-<DT>model_table</DT>
- <DD>TEXT. Name of the table containing the model
- to use for prediction.
- </DD>
-
- <DT>test_table</DT>
- <DD>TEXT. Name of the table containing the dataset to
- predict on. Note that test data is not preprocessed (unlike
- fit and evaluate) so put one test image per row for prediction.
- Also see the comment below for the 'independent_varname' parameter
- regarding normalization.
-
- </DD>
-
- <DT>id_col</DT>
- <DD>TEXT. Name of the id column in the test data table.
- </DD>
-
- <DT>independent_varname</DT>
- <DD>TEXT. Column with independent variables in the test table.
- If a 'normalizing_const' is specified when preprocessing the
- training dataset, this same normalization will be applied to
- the independent variables used in predict.
- </DD>
-
- <DT>output_table</DT>
- <DD>TEXT. Name of the table that prediction output will be
- written to. Table contains:</DD>
- <table class="output">
- <tr>
- <th>id</th>
- <td>Gives the 'id' for each prediction, corresponding to each row from the test_table.</td>
- </tr>
- <tr>
- <th>estimated_COL_NAME</th>
- <td>
- (For pred_type='response') The estimated class
- for classification, where
- COL_NAME is the name of the column to be
- predicted from test data.
- </td>
- </tr>
- <tr>
- <th>prob_CLASS</th>
- <td>
- (For pred_type='prob' for classification) The
- probability of a given class.
- There will be one column for each class
- in the training data.
- </td>
- </tr>
-
- <DT>pred_type (optional)</DT>
- <DD>TEXT, default: 'response'. The type of output
- desired, where 'response' gives the actual prediction
- and 'prob' gives the probability value for each class.
- </DD>
-
- <DT>use_gpus (optional)</DT>
- <DD>BOOLEAN, default: FALSE (i.e., CPU).
- Flag to enable GPU support for training neural network.
- The number of GPUs to use is determined by the parameters
- passed to the preprocessor.
-
- @note
- We have seen some memory related issues when segments
- share GPU resources.
- For example, if you provide 1 GPU and your
- database cluster is set up to have 4
- segments per segment host, it means that all 4
- segments on a segment host will share the same
- GPU. The current recommended
- configuration is 1 GPU per segment.
- </DD>
-
- <DT>mst_key (optional)</DT>
- <DD>INTEGER, default: NULL. ID that defines a unique tuple for
- model architecture-compile parameters-fit parameters in a model
- selection table. Do not use this if training one model at a time using madlib_keras_fit().
- See the <a href="group__grp__keras__run__model__selection.html">Model Selection</a> section
- for more details on model selection by training multiple models at a time.
- </DD>
-</DL>
-
-
-@anchor keras_predict_byom
-@par Predict BYOM (bring your own model)
-The predict BYOM function allows you to do inference on models that
-have not been trained on MADlib, but rather imported from elsewhere.
-It has the following format:
-<pre class="syntax">
-madlib_keras_predict_byom(
- model_arch_table,
- model_id,
- test_table,
- id_col,
- independent_varname,
- output_table,
- pred_type,
- use_gpus,
- class_values,
- normalizing_const
- )
-</pre>
-
-
-\b Arguments
-<dl class="arglist">
-
-<DT>model_arch_table</DT>
- <DD>TEXT. Name of the architecture table containing the model
- to use for prediction. The model weights and architecture can be loaded to
- this table by using the
- <a href="group__grp__keras__model__arch.html">load_keras_model</a> function.
- </DD>
-
- <DT>model_id</DT>
- <DD>INTEGER. This is the id in 'model_arch_table' containing the model
- architecture and model weights to use for prediction.
- </DD>
-
- <DT>test_table</DT>
- <DD>TEXT. Name of the table containing the dataset to
- predict on. Note that test data is not preprocessed (unlike
- fit and evaluate) so put one test image per row for prediction.
- Set the 'normalizing_const' below for the independent variable if necessary.
- </DD>
-
- <DT>id_col</DT>
- <DD>TEXT. Name of the id column in the test data table.
- </DD>
-
- <DT>independent_varname</DT>
- <DD>TEXT. Column with independent variables in the test table.
- Set the 'normalizing_const' below if necessary.
- </DD>
-
- <DT>output_table</DT>
- <DD>TEXT. Name of the table that prediction output will be
- written to. Table contains:</DD>
- <table class="output">
- <tr>
- <th>id</th>
- <td>Gives the 'id' for each prediction, corresponding to each row from the 'test_table'.</td>
- </tr>
- <tr>
- <th>estimated_dependent_var</th>
- <td>
- (For pred_type='response') Estimated class for classification. If
- the 'class_values' parameter is passed in as NULL, then we assume that the class
- labels are [0,1,2...,n-1] where n-1 is the number of classes in the model
- architecture.
- </td>
- </tr>
- <tr>
- <th>prob_CLASS</th>
- <td>
- (For pred_type='prob' for classification)
- Probability of a given class.
- If 'class_values' is passed in as NULL, we create one column called
- 'prob' which is an array of probabilities for each class.
- If 'class_values' is not NULL, then there will be one
- column for each class.
- </td>
- </tr>
-
- <DT>pred_type (optional)</DT>
- <DD>TEXT, default: 'response'. The type of output desired, where 'response'
- gives the actual prediction and 'prob' gives the probability value for each class.
- </DD>
-
- <DT>use_gpus (optional)</DT>
- <DD>BOOLEAN, default: FALSE (i.e., CPU).
- Flag to enable GPU support for training neural network.
- The number of GPUs to use is determined by the parameters
- passed to the preprocessor.
-
- @note
- We have seen some memory related issues when segments
- share GPU resources.
- For example, if you provide 1 GPU and your
- database cluster is set up to have 4
- segments per segment host, it means that all 4
- segments on a segment host will share the same
- GPU. The current recommended
- configuration is 1 GPU per segment.
- </DD>
-
- <DT>class_values (optional)</DT>
- <DD>TEXT[], default: NULL.
- List of class labels that were used while training the model. See the 'output_table'
- column above for more details.
-
- @note
- If you specify the class values parameter,
- it must reflect how the dependent variable was 1-hot encoded for training. If you accidently
- pick another order that does not match the 1-hot encoding, the predictions would be wrong.
- </DD>
-
- <DT>normalizing_const (optional)</DT>
- <DD>DOUBLE PRECISION, default: 1.0.
- The normalizing constant to divide each value in the 'independent_varname'
- array by. For example, you would use 255 for this value if the image data is
- in the form 0-255.
- </DD>
-</DL>
-
-
-@anchor example
-@par Examples
-
-@note
-Deep learning works best on very large datasets,
-but that is not convenient for a quick introduction
-to the syntax. So in this example we use an MLP on the well
-known iris data set from https://archive.ics.uci.edu/ml/datasets/iris.
-For more realistic examples with images please refer
-to the deep learning notebooks
-at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts.
-
-<h4>Classification</h4>
-
--# Create an input data set.
-<pre class="example">
-DROP TABLE IF EXISTS iris_data;
-CREATE TABLE iris_data(
- id serial,
- attributes numeric[],
- class_text varchar
-);
-INSERT INTO iris_data(id, attributes, class_text) VALUES
-(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'),
-(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'),
-(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'),
-(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'),
-(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'),
-(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'),
-(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'),
-(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'),
-(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'),
-(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'),
-(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'),
-(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'),
-(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'),
-(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'),
-(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'),
-(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'),
-(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'),
-(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'),
-(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'),
-(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'),
-(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'),
-(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'),
-(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'),
-(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'),
-(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'),
-(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'),
-(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'),
-(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'),
-(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'),
-(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'),
-(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'),
-(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'),
-(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'),
-(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'),
-(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'),
-(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'),
-(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'),
-(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'),
-(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'),
-(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'),
-(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'),
-(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'),
-(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'),
-(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'),
-(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'),
-(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'),
-(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'),
-(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'),
-(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'),
-(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'),
-(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'),
-(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'),
-(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'),
-(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'),
-(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'),
-(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'),
-(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'),
-(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'),
-(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'),
-(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'),
-(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'),
-(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'),
-(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'),
-(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'),
-(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'),
-(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'),
-(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'),
-(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'),
-(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'),
-(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'),
-(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'),
-(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'),
-(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'),
-(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'),
-(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'),
-(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'),
-(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'),
-(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'),
-(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'),
-(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'),
-(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'),
-(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'),
-(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'),
-(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'),
-(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'),
-(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'),
-(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'),
-(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'),
-(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'),
-(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'),
-(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'),
-(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'),
-(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'),
-(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'),
-(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'),
-(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'),
-(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'),
-(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'),
-(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'),
-(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'),
-(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'),
-(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'),
-(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'),
-(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'),
-(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'),
-(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'),
-(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'),
-(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'),
-(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'),
-(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'),
-(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'),
-(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'),
-(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'),
-(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'),
-(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'),
-(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'),
-(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'),
-(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'),
-(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'),
-(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'),
-(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'),
-(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'),
-(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'),
-(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'),
-(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'),
-(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'),
-(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'),
-(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'),
-(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'),
-(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'),
-(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'),
-(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'),
-(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'),
-(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'),
-(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'),
-(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'),
-(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'),
-(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'),
-(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'),
-(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'),
-(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'),
-(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'),
-(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'),
-(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'),
-(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica');
-</pre>
-Create a test/validation dataset from the training data:
-<pre class="example">
-DROP TABLE IF EXISTS iris_train, iris_test;
--- Set seed so results are reproducible
-SELECT setseed(0);
-SELECT madlib.train_test_split('iris_data', -- Source table
- 'iris', -- Output table root name
- 0.8, -- Train proportion
- NULL, -- Test proportion (0.2)
- NULL, -- Strata definition
- NULL, -- Output all columns
- NULL, -- Sample without replacement
- TRUE -- Separate output tables
- );
-SELECT COUNT(*) FROM iris_train;
-</pre>
-<pre class="result">
- count
-------+
- 120
-</pre>
-
--# Call the preprocessor for deep learning. For the training dataset:
-<pre class="example">
-\\x off
-DROP TABLE IF EXISTS iris_train_packed, iris_train_packed_summary;
-SELECT madlib.training_preprocessor_dl('iris_train', -- Source table
- 'iris_train_packed', -- Output table
- 'class_text', -- Dependent variable
- 'attributes' -- Independent variable
- );
-SELECT * FROM iris_train_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------+---------------------------------------------
-source_table | iris_train
-output_table | iris_train_packed
-dependent_varname | class_text
-independent_varname | attributes
-dependent_vartype | character varying
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size | 60
-normalizing_const | 1.0
-num_classes | 3
-</pre>
-For the validation dataset:
-<pre class="example">
-DROP TABLE IF EXISTS iris_test_packed, iris_test_packed_summary;
-SELECT madlib.validation_preprocessor_dl('iris_test', -- Source table
- 'iris_test_packed', -- Output table
- 'class_text', -- Dependent variable
- 'attributes', -- Independent variable
- 'iris_train_packed' -- From training preprocessor step
- );
-SELECT * FROM iris_test_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------+---------------------------------------------
-source_table | iris_test
-output_table | iris_test_packed
-dependent_varname | class_text
-independent_varname | attributes
-dependent_vartype | character varying
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size | 15
-normalizing_const | 1.0
-num_classes | 3
-</pre>
-
--# Define and load model architecture. Use Keras to define
-the model architecture:
-<pre class="example">
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model_simple = Sequential()
-model_simple.add(Dense(10, activation='relu', input_shape=(4,)))
-model_simple.add(Dense(10, activation='relu'))
-model_simple.add(Dense(3, activation='softmax'))
-model_simple.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type) Output Shape Param #
-=================================================================
-dense_1 (Dense) (None, 10) 50
-_________________________________________________________________
-dense_2 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_3 (Dense) (None, 3) 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-</pre>
-Export the model to JSON:
-<pre class="example">
-model_simple.to_json()
-</pre>
-<pre class="result">
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-</pre>
-Load into model architecture table:
-<pre class="example">
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Sophie', -- Name
- 'A simple model' -- Descr
-);
-</pre>
-
--# Train model and view summary table:
-<pre class="example">
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
- 'iris_model', -- model output table
- 'model_arch_library', -- model arch table
- 1, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 10 -- num_iterations
- );
-SELECT * FROM iris_model_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------
-source_table | iris_train_packed
-model | iris_model
-dependent_varname | class_text
-independent_varname | attributes
-model_arch_table | model_arch_library
-model_id | 1
-compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params | batch_size=5, epochs=3
-num_iterations | 10
-validation_table |
-metrics_compute_frequency | 10
-name |
-description |
-model_type | madlib_keras
-model_size | 0.7900390625
-start_training_time | 2019-06-05 20:55:15.785034
-end_training_time | 2019-06-05 20:55:25.373035
-metrics_elapsed_time | {9.58799290657043}
-madlib_version | 1.17.0
-num_classes | 3
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype | character varying
-normalizing_const | 1
-metrics_type | {accuracy}
-training_metrics_final | 0.766666650772
-training_loss_final | 0.721103310585
-training_metrics | {0.766666650772095}
-training_loss | {0.721103310585022}
-validation_metrics_final |
-validation_loss_final |
-validation_metrics |
-validation_loss |
-metrics_iters | {10}
-</pre>
-
--# Use the test dataset to evaluate the model we built above:
-<pre class="example">
-DROP TABLE IF EXISTS iris_validate;
-SELECT madlib.madlib_keras_evaluate('iris_model', -- model
- 'iris_test_packed', -- test table
- 'iris_validate' -- output table
- );
-SELECT * FROM iris_validate;
-</pre>
-<pre class="result">
- loss | metric | metrics_type
--------------------+-------------------+--------------
- 0.719491899013519 | 0.800000011920929 | {accuracy}
-(1 row)
-</pre>
-
--# Predict. We will use the validation dataset for prediction
-as well, which is not usual but serves to show the
-syntax. The prediction is in the 'estimated_class_text'
-column:
-<pre class="example">
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_model', -- model
- 'iris_test', -- test_table
- 'id', -- id column
- 'attributes', -- independent var
- 'iris_predict' -- output table
- );
-SELECT * FROM iris_predict ORDER BY id;
-</pre>
-<pre class="result">
- id | estimated_class_text
------+----------------------
- 4 | Iris-setosa
- 6 | Iris-setosa
- 8 | Iris-setosa
- 12 | Iris-setosa
- 13 | Iris-setosa
- 15 | Iris-setosa
- 24 | Iris-setosa
- 30 | Iris-setosa
- 38 | Iris-setosa
- 49 | Iris-setosa
- 60 | Iris-virginica
- 68 | Iris-versicolor
- 69 | Iris-versicolor
- 76 | Iris-versicolor
- 78 | Iris-versicolor
- 81 | Iris-versicolor
- 85 | Iris-virginica
- 90 | Iris-versicolor
- 91 | Iris-versicolor
- 94 | Iris-virginica
- 104 | Iris-virginica
- 106 | Iris-versicolor
- 107 | Iris-virginica
- 110 | Iris-virginica
- 119 | Iris-versicolor
- 127 | Iris-virginica
- 129 | Iris-virginica
- 134 | Iris-versicolor
- 139 | Iris-virginica
- 144 | Iris-virginica
-(30 rows)
-</pre>
-Count missclassifications:
-<pre class="example">
-SELECT COUNT(*) FROM iris_predict JOIN iris_test USING (id)
-WHERE iris_predict.estimated_class_text != iris_test.class_text;
-</pre>
-<pre class="result">
- count
--------+
- 6
-(1 row)
-</pre>
-Accuracy:
-<pre class="example">
-SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from
- (select iris_test.class_text as actual, iris_predict.estimated_class_text as estimated
- from iris_predict inner join iris_test
- on iris_test.id=iris_predict.id) q
-WHERE q.actual=q.estimated;
-</pre>
-<pre class="result">
- test_accuracy_percent
------------------------+
- 80.00
-(1 row)
-</pre>
-
--# Predict BYOM.
-We will use the validation dataset for prediction
-as well, which is not usual but serves to show the
-syntax. See <a href="group__grp__keras__model__arch.html">load_keras_model</a>
-for details on how to load the model architecture and weights.
-In this example we will use weights we already have:
-<pre class="example">
-UPDATE model_arch_library
-SET model_weights = iris_model.model_weights
-FROM iris_model
-WHERE model_arch_library.model_id = 1;
-</pre>
-Now train using a model from the model architecture table directly
-without referencing the model table from the MADlib training. Note that if you
-specify the class values parameter as we do below, it must reflect how the dependent
-variable was 1-hot encoded for training. In this example the 'training_preprocessor_dl()'
-in Step 2 above encoded in the order {'Iris-setosa', 'Iris-versicolor', 'Iris-virginica'} so
-this is the order we pass in the parameter. If we accidently pick another order that does
-not match the 1-hot encoding, the predictions would be wrong.
-<pre class="example">
-DROP TABLE IF EXISTS iris_predict_byom;
-SELECT madlib.madlib_keras_predict_byom('model_arch_library', -- model arch table
- 1, -- model arch id
- 'iris_test', -- test_table
- 'id', -- id column
- 'attributes', -- independent var
- 'iris_predict_byom', -- output table
- 'response', -- prediction type
- FALSE, -- use GPUs
- ARRAY['Iris-setosa', 'Iris-versicolor', 'Iris-virginica'], -- class values
- 1.0 -- normalizing const
- );
-SELECT * FROM iris_predict_byom ORDER BY id;
-</pre>
-The prediction is in the 'estimated_dependent_var' column:
-<pre class="result">
- id | estimated_dependent_var
------+----------------------
- 4 | Iris-setosa
- 6 | Iris-setosa
- 8 | Iris-setosa
- 12 | Iris-setosa
- 13 | Iris-setosa
- 15 | Iris-setosa
- 24 | Iris-setosa
- 30 | Iris-setosa
- 38 | Iris-setosa
- 49 | Iris-setosa
- 60 | Iris-virginica
- 68 | Iris-versicolor
- 69 | Iris-versicolor
- 76 | Iris-versicolor
- 78 | Iris-versicolor
- 81 | Iris-versicolor
- 85 | Iris-virginica
- 90 | Iris-versicolor
- 91 | Iris-versicolor
- 94 | Iris-virginica
- 104 | Iris-virginica
- 106 | Iris-versicolor
- 107 | Iris-virginica
- 110 | Iris-virginica
- 119 | Iris-versicolor
- 127 | Iris-virginica
- 129 | Iris-virginica
- 134 | Iris-versicolor
- 139 | Iris-virginica
- 144 | Iris-virginica
-(30 rows)
- </pre>
-Count missclassifications:
-<pre class="example">
-SELECT COUNT(*) FROM iris_predict_byom JOIN iris_test USING (id)
-WHERE iris_predict_byom.estimated_dependent_var != iris_test.class_text;
-</pre>
-<pre class="result">
- count
--------+
- 6
-(1 row)
-</pre>
-Accuracy:
-<pre class="example">
-SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from
- (select iris_test.class_text as actual, iris_predict_byom.estimated_dependent_var as estimated
- from iris_predict_byom inner join iris_test
- on iris_test.id=iris_predict_byom.id) q
-WHERE q.actual=q.estimated;
-</pre>
-<pre class="result">
- test_accuracy_percent
------------------------+
- 80.00
-(1 row)
-</pre>
-
-
-<h4>Classification with Other Parameters</h4>
-
--# Validation dataset. Now use a validation dataset
-and compute metrics every 3rd iteration using
-the 'metrics_compute_frequency' parameter. This can
-help reduce run time if you do not need metrics
-computed at every iteration.
-<pre class="example">
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
- 'iris_model', -- model output table
- 'model_arch_library', -- model arch table
- 1, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 10, -- num_iterations
- FALSE, -- use GPUs
- 'iris_test_packed', -- validation dataset
- 3, -- metrics compute frequency
- FALSE, -- warm start
- 'Sophie L.', -- name
- 'Simple MLP for iris dataset' -- description
- );
-SELECT * FROM iris_model_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------
-source_table | iris_train_packed
-model | iris_model
-dependent_varname | class_text
-independent_varname | attributes
-model_arch_table | model_arch_library
-model_id | 1
-compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params | batch_size=5, epochs=3
-num_iterations | 10
-validation_table | iris_test_packed
-metrics_compute_frequency | 3
-name | Sophie L.
-description | Simple MLP for iris dataset
-model_type | madlib_keras
-model_size | 0.7900390625
-start_training_time | 2019-06-05 20:58:23.224629
-end_training_time | 2019-06-05 20:58:35.477499
-metrics_elapsed_time | {4.69859290122986,8.2062520980835,10.8104848861694,12.2528700828552}
-madlib_version | 1.17.0
-num_classes | 3
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype | character varying
-normalizing_const | 1
-metrics_type | {accuracy}
-training_metrics_final | 0.941666662693
-training_loss_final | 0.40586027503
-training_metrics | {0.699999988079071,0.800000011920929,0.899999976158142,0.941666662693024}
-training_loss | {0.825238645076752,0.534248650074005,0.427499741315842,0.405860275030136}
-validation_metrics_final | 0.866666674614
-validation_loss_final | 0.409001916647
-validation_metrics | {0.733333349227905,0.733333349227905,0.866666674613953,0.866666674613953}
-validation_loss | {0.827081918716431,0.536275088787079,0.431326270103455,0.409001916646957}
-metrics_iters | {3,6,9,10}
-</pre>
-
--# Predict probabilities for each class:
-<pre class="example">
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_model', -- model
- 'iris_test', -- test_table
- 'id', -- id column
- 'attributes', -- independent var
- 'iris_predict', -- output table
- 'prob' -- response type
- );
-SELECT * FROM iris_predict ORDER BY id;
-</pre>
-<pre class="result">
- id | prob_Iris-setosa | prob_Iris-versicolor | prob_Iris-virginica
------+------------------+----------------------+---------------------
- 4 | 0.9241953 | 0.059390426 | 0.01641435
- 6 | 0.9657151 | 0.02809224 | 0.0061926916
- 8 | 0.9543316 | 0.03670931 | 0.008959154
- 12 | 0.93851465 | 0.048681837 | 0.012803554
- 13 | 0.93832576 | 0.04893658 | 0.012737647
- 15 | 0.98717564 | 0.01091238 | 0.0019119986
- 24 | 0.9240628 | 0.060805064 | 0.015132156
- 30 | 0.92063266 | 0.062279057 | 0.017088294
- 38 | 0.9353765 | 0.051353406 | 0.013270103
- 49 | 0.9709265 | 0.023811856 | 0.005261566
- 60 | 0.034395564 | 0.5260507 | 0.43955377
- 68 | 0.031360663 | 0.53689945 | 0.43173987
- 69 | 0.0098787155 | 0.46121457 | 0.52890676
- 76 | 0.031186827 | 0.5644549 | 0.40435827
- 78 | 0.00982633 | 0.48929632 | 0.5008774
- 81 | 0.03658528 | 0.53248984 | 0.4309249
- 85 | 0.015423619 | 0.48452598 | 0.5000504
- 90 | 0.026857043 | 0.5155698 | 0.45757324
- 91 | 0.013675574 | 0.47155368 | 0.5147708
- 94 | 0.073440716 | 0.5418821 | 0.3846772
- 104 | 0.0021637122 | 0.3680499 | 0.62978643
- 106 | 0.00052832486 | 0.30891812 | 0.6905536
- 107 | 0.007315576 | 0.40949163 | 0.5831927
- 110 | 0.0022259138 | 0.4058138 | 0.59196025
- 119 | 0.00018505375 | 0.24510723 | 0.7547077
- 127 | 0.009542585 | 0.46958733 | 0.52087003
- 129 | 0.0019719477 | 0.36288205 | 0.635146
- 134 | 0.0056418083 | 0.43401477 | 0.56034344
- 139 | 0.01067015 | 0.4755573 | 0.51377255
- 144 | 0.0018909549 | 0.37689638 | 0.6212126
-(30 rows)
-</pre>
-
--# Warm start. Next, use the warm_start parameter
-to continue learning, using the coefficients from
-the run above. Note that we don't drop the
-model table or model summary table:
-<pre class="example">
-SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
- 'iris_model', -- model output table
- 'model_arch_library', -- model arch table
- 1, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 5, -- num_iterations
- FALSE, -- use GPUs
- 'iris_test_packed', -- validation dataset
- 1, -- metrics compute frequency
- TRUE, -- warm start
- 'Sophie L.', -- name
- 'Simple MLP for iris dataset' -- description
- );
-SELECT * FROM iris_model_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------------------------
-source_table | iris_train_packed
-model | iris_model
-dependent_varname | class_text
-independent_varname | attributes
-model_arch_table | model_arch_library
-model_id | 1
-compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params | batch_size=5, epochs=3
-num_iterations | 5
-validation_table | iris_test_packed
-metrics_compute_frequency | 1
-name | Sophie L.
-description | Simple MLP for iris dataset
-model_type | madlib_keras
-model_size | 0.7900390625
-start_training_time | 2019-06-05 20:59:43.971792
-end_training_time | 2019-06-05 20:59:51.654586
-metrics_elapsed_time | {2.89326310157776,4.14273309707642,5.24781513214111,6.34498596191406,7.68279695510864}
-madlib_version | 1.17.0
-num_classes | 3
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype | character varying
-normalizing_const | 1
-metrics_type | {accuracy}
-training_metrics_final | 0.933333337307
-training_loss_final | 0.334455043077
-training_metrics | {0.933333337306976,0.933333337306976,0.975000023841858,0.975000023841858,0.933333337306976}
-training_loss | {0.386842548847198,0.370587915182114,0.357161343097687,0.344598710536957,0.334455043077469}
-validation_metrics_final | 0.866666674614
-validation_loss_final | 0.34414178133
-validation_metrics | {0.866666674613953,0.866666674613953,0.933333337306976,0.866666674613953,0.866666674613953}
-validation_loss | {0.391442179679871,0.376414686441422,0.362262904644012,0.351912915706635,0.344141781330109}
-metrics_iters | {1,2,3,4,5}
-</pre>
-Note that the loss and accuracy values pick up from where the previous run left off.
-
-<h4>Transfer Learning</h4>
-Here we want
-to start with initial weights from a pre-trained model
-rather than training from scratch. We also want to use
-a model architecture with the earlier feature layer(s)
-frozen to save on training time. The example below is
-somewhat contrived but gives you the idea of the steps.
-
--# Define and load a model architecture with
-the 1st hidden layer frozen:
-
-<pre class="example">
-model_transfer = Sequential()
-model_transfer.add(Dense(10, activation='relu', input_shape=(4,), trainable=False))
-model_transfer.add(Dense(10, activation='relu'))
-model_transfer.add(Dense(3, activation='softmax'))
-model_simple.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type) Output Shape Param #
-=================================================================
-dense_1 (Dense) (None, 10) 50
-_________________________________________________________________
-dense_2 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_3 (Dense) (None, 3) 33
-=================================================================
-Total params: 193
-Trainable params: 143
-Non-trainable params: 50
-\endverbatim
-</pre>
-Export the model to JSON:
-<pre class="example">
-model_simple.to_json()
-</pre>
-<pre class="result">
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": false, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-</pre>
-Load into model architecture table:
-<pre class="example">
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": false, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Maria', -- Name
- 'A transfer model' -- Descr
-);
-</pre>
-Fetch the weights from a previous MADlib run. (Normally
-these would be downloaded from a source that trained
-the same model architecture on a related dataset.)
-<pre class="example">
-UPDATE model_arch_library
-SET model_weights = iris_model.model_weights
-FROM iris_model
-WHERE model_arch_library.model_id = 2;
-</pre>
-Now train the model using the transfer model and the pre-trained weights:
-<pre class="example">
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
- 'iris_model', -- model output table
- 'model_arch_library', -- model arch table
- 2, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 10 -- num_iterations
- );
-SELECT * FROM iris_model_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------
-source_table | iris_train_packed
-model | iris_model
-dependent_varname | class_text
-independent_varname | attributes
-model_arch_table | model_arch_library
-model_id | 2
-compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params | batch_size=5, epochs=3
-num_iterations | 10
-validation_table |
-metrics_compute_frequency | 10
-name |
-description |
-model_type | madlib_keras
-model_size | 0.7900390625
-start_training_time | 2019-06-05 21:01:03.998422
-end_training_time | 2019-06-05 21:01:13.525838
-metrics_elapsed_time | {9.52741599082947}
-madlib_version | 1.17.0
-num_classes | 3
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype | character varying
-normalizing_const | 1
-metrics_type | {accuracy}
-training_metrics_final | 0.975000023842
-training_loss_final | 0.245171800256
-training_metrics | {0.975000023841858}
-training_loss | {0.245171800255775}
-validation_metrics_final |
-validation_loss_final |
-validation_metrics |
-validation_loss |
-metrics_iters | {10}
-</pre>
-
-@anchor notes
-@par Notes
-
-1. Refer to the deep learning section of the Apache MADlib
-wiki [5] for important information including supported libraries
-and versions.
-
-2. Classification is currently supported, not regression.
-
-3. Reminder about the distinction between warm start and transfer learning. Warm start uses model
-state (weights) from the model output table from a previous training run -
-set the 'warm_start' parameter to TRUE in the fit function.
-Transfer learning uses initial model state (weights) stored in the 'model_arch_table' - in this case set the
-'warm_start' parameter to FALSE in the fit function.
-
-@anchor background
-@par Technical Background
-
-For an introduction to deep learning foundations, including MLP and CNN,
-refer to [6].
-
-This module trains a single large model across the database cluster
-using the bulk synchronous parallel (BSP) approach, with model averaging [7].
-
-On the effect of database cluster size: as the database cluster size increases, the per iteration
-loss will be higher since the model only sees 1/n of the data, where n is the number of segments.
-However, each iteration runs faster than single node because it is only traversing 1/n of the data.
-For highly non-convex solution spaces, convergence behavior may diminish as cluster size increases.
-Ensure that each segment has sufficient volume of data and examples of each class value.
-
-Alternatively, to train multiple models at the same time for model
-architecture search or hyperparameter tuning, you can
-use <a href="group__grp__keras__run__model__selection.html">Model Selection</a>,
-which does not do model averaging and hence may have better covergence efficiency.
-
-@anchor literature
-@literature
-
-@anchor mlp-lit-1
-[1] https://keras.io/
-
-[2] https://www.tensorflow.org/
-
-[3] "Neural Networks for Machine Learning", Lectures 6a and 6b on mini-batch gradient descent,
-Geoffrey Hinton with Nitish Srivastava and Kevin Swersky,
-http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf
-
-[4] https://keras.io/models/model/
-
-[5] Deep learning section of Apache MADlib wiki, https://cwiki.apache.org/confluence/display/MADLIB/Deep+Learning
-
-[6] Deep Learning, Ian Goodfellow, Yoshua Bengio and Aaron Courville, MIT Press, 2016.
-
-[7] "Resource-Efficient and Reproducible Model Selection on Deep Learning Systems," Supun Nakandala,
-Yuhao Zhang, and Arun Kumar, Technical Report, Computer Science and Engineering, University of California,
-San Diego https://adalabucsd.github.io/papers/TR_2019_Cerebro.pdf.
-
-[8] Greenplum Database server configuration parameters https://gpdb.docs.pivotal.io/latest/ref_guide/config_params/guc-list.html
-
-@anchor related
-@par Related Topics
-
-File madlib_keras.sql_in documenting the training, evaluate and predict functions.
-
-*/
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN,
- name VARCHAR,
- description VARCHAR
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras')
- from utilities.control import SetGUC
- with AOControl(False):
- with SetGUC("plan_cache_mode", "force_generic_plan"):
- madlib_keras.fit(**globals())
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN,
- name VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER
-) RETURNS VOID AS $$
-SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
- m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- source_table VARCHAR,
- model VARCHAR,
- model_arch_table VARCHAR,
- model_id INTEGER,
- compile_params VARCHAR,
- fit_params VARCHAR,
- num_iterations INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, FALSE, NULL, NULL, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_transition(
- state BYTEA,
- dependent_var BYTEA,
- independent_var BYTEA,
- dependent_var_shape INTEGER[],
- independent_var_shape INTEGER[],
- model_architecture TEXT,
- compile_params TEXT,
- fit_params TEXT,
- dist_key INTEGER,
- dist_key_mapping INTEGER[],
- current_seg_id INTEGER,
- segments_per_host INTEGER,
- images_per_seg INTEGER[],
- use_gpus BOOLEAN,
- accessible_gpus_for_seg INTEGER[],
- prev_serialized_weights BYTEA,
- is_final_iteration BOOLEAN
-) RETURNS BYTEA AS $$
-PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
- return madlib_keras.fit_transition(**globals())
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_merge(
- state1 BYTEA,
- state2 BYTEA
-) RETURNS BYTEA AS $$
-PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
- return madlib_keras.fit_merge(**globals())
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_final(
- state BYTEA
-) RETURNS BYTEA AS $$
-PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
- return madlib_keras.fit_final(**globals())
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fit_step(
- BYTEA,
- BYTEA,
- TEXT,
- TEXT,
- TEXT,
- TEXT,
- TEXT,
- INTEGER,
- INTEGER[],
- INTEGER,
- INTEGER,
- INTEGER[],
- BOOLEAN,
- INTEGER[],
- BYTEA,
- BOOLEAN);
-CREATE AGGREGATE MADLIB_SCHEMA.fit_step(
- /* dep_var */ BYTEA,
- /* ind_var */ BYTEA,
- /* dep_var_shape */ INTEGER[],
- /* ind_var_shape */ INTEGER[],
- /* model_architecture */ TEXT,
- /* compile_params */ TEXT,
- /* fit_params */ TEXT,
- /* dist_key */ INTEGER,
- /* dist_key_mapping */ INTEGER[],
- /* current_seg_id */ INTEGER,
- /* segments_per_host */ INTEGER,
- /* images_per_seg */ INTEGER[],
- /* use_gpus */ BOOLEAN,
- /* segments_per_host */ INTEGER[],
- /* serialized_weights */ BYTEA,
- /* is_final_iteration */ BOOLEAN
-)(
- STYPE=BYTEA,
- SFUNC=MADLIB_SCHEMA.fit_transition,
- m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.fit_merge,')
- FINALFUNC=MADLIB_SCHEMA.fit_final
-);
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict(
- model_table VARCHAR,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR,
- pred_type VARCHAR,
- use_gpus BOOLEAN,
- mst_key INTEGER
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras_predict')
- from utilities.control import SetGUC
- with AOControl(False):
- with SetGUC("plan_cache_mode", "force_generic_plan"):
- madlib_keras_predict.Predict(schema_madlib,
- model_table,
- test_table,
- id_col,
- independent_varname,
- output_table,
- pred_type,
- use_gpus,
- mst_key)
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict(
- model_table VARCHAR,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR,
- pred_type VARCHAR,
- use_gpus BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict($1, $2, $3, $4, $5, $6, $7, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict(
- model_table VARCHAR,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR,
- pred_type VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict($1, $2, $3, $4, $5, $6, FALSE, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict(
- model_table VARCHAR,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict($1, $2, $3, $4, $5, NULL, FALSE, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_predict(
- independent_var REAL[],
- model_architecture TEXT,
- model_weights BYTEA,
- is_response BOOLEAN,
- normalizing_const DOUBLE PRECISION,
- current_seg_id INTEGER,
- seg_ids INTEGER[],
- images_per_seg INTEGER[],
- use_gpus BOOLEAN,
- gpus_per_host INTEGER,
- segments_per_host INTEGER
-) RETURNS DOUBLE PRECISION[] AS $$
- PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras_predict')
- return madlib_keras_predict.internal_keras_predict(**globals())
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
--------------------------------------------------------------------------------
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
- model_arch_table VARCHAR,
- model_id INTEGER,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR,
- pred_type VARCHAR,
- use_gpus BOOLEAN,
- class_values TEXT[],
- normalizing_const DOUBLE PRECISION
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras_predict')
- from utilities.control import SetGUC
- with AOControl(False):
- with SetGUC("plan_cache_mode", "force_generic_plan"):
- madlib_keras_predict.PredictBYOM(**globals())
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
- model_arch_table VARCHAR,
- model_id INTEGER,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR,
- pred_type VARCHAR,
- use_gpus BOOLEAN,
- class_values TEXT[]
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
- model_arch_table VARCHAR,
- model_id INTEGER,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR,
- pred_type VARCHAR,
- use_gpus BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
- model_arch_table VARCHAR,
- model_id INTEGER,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR,
- pred_type VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, $7, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
- model_arch_table VARCHAR,
- model_id INTEGER,
- test_table VARCHAR,
- id_col VARCHAR,
- independent_varname VARCHAR,
- output_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, NULL, NULL, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
--------------------------------------------------------------------------------
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate(
- model_table VARCHAR,
- test_table VARCHAR,
- output_table VARCHAR,
- use_gpus BOOLEAN,
- mst_key INTEGER
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras')
- from utilities.control import SetGUC
- with AOControl(False):
- with SetGUC("plan_cache_mode", "force_generic_plan"):
- madlib_keras.evaluate(**globals())
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate(
- model_table VARCHAR,
- test_table VARCHAR,
- output_table VARCHAR,
- use_gpus BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_evaluate($1, $2, $3, $4, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate(
- model_table VARCHAR,
- test_table VARCHAR,
- output_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_evaluate($1, $2, $3, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_transition(
- state REAL[3],
- dependent_var BYTEA,
- independent_var BYTEA,
- dependent_var_shape INTEGER[],
- independent_var_shape INTEGER[],
- model_architecture TEXT,
- serialized_weights BYTEA,
- compile_params TEXT,
- dist_key INTEGER,
- dist_key_mapping INTEGER[],
- current_seg_id INTEGER,
- segments_per_host INTEGER,
- images_per_seg INTEGER[],
- use_gpus BOOLEAN,
- accessible_gpus_for_seg INTEGER[],
- is_final_iteration BOOLEAN
-) RETURNS REAL[3] AS $$
-PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
- return madlib_keras.internal_keras_eval_transition(**globals())
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_merge(
- state1 REAL[3],
- state2 REAL[3]
-) RETURNS REAL[3] AS $$
-PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
- return madlib_keras.internal_keras_eval_merge(**globals())
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_final(
- state REAL[3]
-) RETURNS REAL[2] AS $$
-PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
- return madlib_keras.internal_keras_eval_final(**globals())
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.internal_keras_evaluate(
- BYTEA,
- BYTEA,
- INTEGER[],
- INTEGER[],
- TEXT,
- BYTEA,
- TEXT,
- INTEGER,
- INTEGER[],
- INTEGER,
- INTEGER,
- INTEGER[],
- BOOLEAN,
- INTEGER[],
- BOOLEAN);
-
-CREATE AGGREGATE MADLIB_SCHEMA.internal_keras_evaluate(
- /* dependent_var */ BYTEA,
- /* independent_var */ BYTEA,
- /* dependent_var_shape */ INTEGER[],
- /* independent_var_shape */ INTEGER[],
- /* model_architecture */ TEXT,
- /* model_weights */ BYTEA,
- /* compile_params */ TEXT,
- /* dist_key */ INTEGER,
- /* dist_key_mapping */ INTEGER[],
- /* current_seg_id */ INTEGER,
- /* segments_per_host */ INTEGER,
- /* images_per_seg*/ INTEGER[],
- /* use_gpus */ BOOLEAN,
- /* accessible_gpus_for_seg */ INTEGER[],
- /* is_final_iteration */ BOOLEAN
-)(
- STYPE=REAL[3],
- INITCOND='{0,0,0}',
- SFUNC=MADLIB_SCHEMA.internal_keras_eval_transition,
- m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.internal_keras_eval_merge,')
- FINALFUNC=MADLIB_SCHEMA.internal_keras_eval_final
-);
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
- message VARCHAR
-) RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras')
- with AOControl(False):
- return madlib_keras.fit_help(**globals())
-$$ LANGUAGE plpythonu IMMUTABLE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit()
-RETURNS VARCHAR AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit('');
-$$ LANGUAGE sql IMMUTABLE
-m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate(
- message VARCHAR
-) RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras')
- with AOControl(False):
- return madlib_keras.evaluate_help(**globals())
-$$ LANGUAGE plpythonu IMMUTABLE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate()
-RETURNS VARCHAR AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_evaluate('');
-$$ LANGUAGE sql IMMUTABLE
-m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict(
- message VARCHAR
-) RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras_predict')
- with AOControl(False):
- return madlib_keras_predict.predict_help(**globals())
-$$ LANGUAGE plpythonu IMMUTABLE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict()
-RETURNS VARCHAR AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_predict('');
-$$ LANGUAGE sql IMMUTABLE
-m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
-
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
deleted file mode 100644
index 9de9774..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
+++ /dev/null
@@ -1,671 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import plpy
-import time
-import sys
-
-from keras.models import *
-from madlib_keras import compute_loss_and_metrics
-from madlib_keras import get_initial_weights
-from madlib_keras import get_model_arch_weights
-from madlib_keras import get_source_summary_table_dict
-from madlib_keras import should_compute_metrics_this_iter
-from madlib_keras_helper import *
-from madlib_keras_model_selection import ModelSelectionSchema
-from madlib_keras_validator import *
-from madlib_keras_wrapper import *
-
-from utilities.control import MinWarning
-from utilities.control import OptimizerControl
-from utilities.control import SetGUC
-from utilities.utilities import add_postfix
-from utilities.utilities import is_platform_gp6_or_up
-from utilities.utilities import unique_string
-from utilities.utilities import rotate
-from utilities.utilities import madlib_version
-from utilities.utilities import is_platform_pg
-from utilities.utilities import get_seg_number
-from utilities.utilities import get_segments_per_host
-from utilities.utilities import rename_table
-
-import json
-from collections import defaultdict
-import random
-import datetime
-mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
-mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
-dist_key_col = DISTRIBUTION_KEY_COLNAME
-
-"""
-FitMultipleModel: This class implements the Model Hopper technique for
-training multiple models in parallel. The goal of this function is to train
-multiple different models on the same data with different parameters.
-The main advantage of this method over running the existing fit function in a
-loop is avoiding inaccuracies caused by the model averaging. The basic idea of
-model hopper is simple. Let's assume that there are n segments and c*n model
-configurations. We begin with distributing these configs to segments. After
-that, each segment trains their c models on the data they have locally for one
-iteration. Once we have these models, we move them to a different segment
-(hopping) as well as receive a different set of models. Once we have the new
-models, we use our segments data to refine them (similar to the warm start
-functionality). Once every model hops through every segment, we consider an
-iteration complete.
-
-This method ensures that we don't have to average any model and the
-loss&accuracy is very close to the ideal case, where all of the data is in one
-segment.
-
-Note that this function is disabled for Postgres.
-"""
-
-@MinWarning("warning")
-class FitMultipleModel():
- def __init__(self, schema_madlib, source_table, model_output_table,
- model_selection_table, num_iterations,
- use_gpus=False, validation_table=None,
- metrics_compute_frequency=None, warm_start=False, name="",
- description="", **kwargs):
- # set the random seed for visit order/scheduling
- random.seed(1)
- if is_platform_pg():
- plpy.error(
- "DL: Multiple model training is not supported on PostgreSQL.")
- self.source_table = source_table
- self.validation_table = validation_table
- self.model_selection_table = model_selection_table
- if self.model_selection_table:
- self.model_selection_summary_table = add_postfix(self.model_selection_table, '_summary')
-
- self.num_iterations = num_iterations
- self.metrics_compute_frequency = metrics_compute_frequency
- self.name = name
- self.description = description
- self.module_name = 'madlib_keras_fit_multiple_model'
- self.schema_madlib = schema_madlib
- self.version = madlib_version(self.schema_madlib)
- self.mst_key_col = ModelSelectionSchema.MST_KEY
- self.model_id_col = ModelSelectionSchema.MODEL_ID
- self.compile_params_col = ModelSelectionSchema.COMPILE_PARAMS
- self.fit_params_col = ModelSelectionSchema.FIT_PARAMS
- self.model_arch_table_col = ModelSelectionSchema.MODEL_ARCH_TABLE
- self.model_weights_col = ModelArchSchema.MODEL_WEIGHTS
- self.model_arch_col = ModelArchSchema.MODEL_ARCH
- self.train_mst_metric_eval_time = defaultdict(list)
- self.train_mst_loss = defaultdict(list)
- self.train_mst_metric = defaultdict(list)
- self.info_str = ""
- self.dep_shape_col = add_postfix(mb_dep_var_col, "_shape")
- self.ind_shape_col = add_postfix(mb_indep_var_col, "_shape")
- self.use_gpus = use_gpus
- self.segments_per_host = get_segments_per_host()
- if self.use_gpus:
- self.accessible_gpus_for_seg = get_accessible_gpus_for_seg(
- self.schema_madlib, self.segments_per_host, self.module_name)
- else:
- self.accessible_gpus_for_seg = get_seg_number()*[0]
-
- self.original_model_output_table = model_output_table
- if self.original_model_output_table:
- self.model_info_table = add_postfix(self.original_model_output_table, '_info')
- self.model_summary_table = add_postfix(
- self.original_model_output_table, '_summary')
-
- self.model_output_table = self.original_model_output_table
-
- """
- For warm start, we need to copy the model output table to a temp table
- because we call truncate on the model output table while training.
- If the query gets aborted, we need to make sure that the user passed
- model output table can be recovered.
- """
- self.warm_start = bool(warm_start)
- self.warm_start_msts = []
- if self.warm_start:
- self.model_output_table = unique_string('initial_model')
-
- self.fit_validator_train = FitMultipleInputValidator(
- self.source_table, self.validation_table, self.original_model_output_table,
- self.model_selection_table, self.model_selection_summary_table,
- mb_dep_var_col, mb_indep_var_col, self.num_iterations,
- self.model_info_table, self.mst_key_col, self.model_arch_table_col,
- self.metrics_compute_frequency, self.warm_start, self.use_gpus,
- self.accessible_gpus_for_seg)
- if self.metrics_compute_frequency is None:
- self.metrics_compute_frequency = num_iterations
-
-
- self.msts = self.fit_validator_train.msts
- self.model_arch_table = self.fit_validator_train.model_arch_table
- self.metrics_iters = []
-
- original_cuda_env = None
- if CUDA_VISIBLE_DEVICES_KEY in os.environ:
- original_cuda_env = os.environ[CUDA_VISIBLE_DEVICES_KEY]
-
- self.dist_key_mapping, self.images_per_seg_train = \
- get_image_count_per_seg_for_minibatched_data_from_db(
- self.source_table)
-
- if self.validation_table:
- self.valid_mst_metric_eval_time = defaultdict(list)
- self.valid_mst_loss = defaultdict(list)
- self.valid_mst_metric = defaultdict(list)
- self.dist_key_mapping_valid, self.images_per_seg_valid = \
- get_image_count_per_seg_for_minibatched_data_from_db(
- self.validation_table)
- self.mst_weights_tbl = unique_string(desp='mst_weights')
- self.mst_current_schedule_tbl = unique_string(desp='mst_current_schedule')
-
- self.dist_keys = query_dist_keys(self.source_table, dist_key_col)
- if len(self.msts) < len(self.dist_keys):
- self.msts_for_schedule = self.msts + [None] * \
- (len(self.dist_keys) - len(self.msts))
- else:
- self.msts_for_schedule = self.msts
- random.shuffle(self.msts_for_schedule)
- self.grand_schedule = self.generate_schedule(self.msts_for_schedule)
- self.gp_segment_id_col = '0' if is_platform_pg() else GP_SEGMENT_ID_COLNAME
- self.unlogged_table = "UNLOGGED" if is_platform_gp6_or_up() else ''
-
- if self.warm_start:
- self.create_model_output_table_warm_start()
- else:
- self.create_model_output_table()
-
- self.weights_to_update_tbl = unique_string(desp='weights_to_update')
- self.fit_multiple_model()
-
- # Update and cleanup metadata tables
- self.insert_info_table()
- self.create_model_summary_table()
- if self.warm_start:
- self.cleanup_for_warm_start()
- reset_cuda_env(original_cuda_env)
-
- def fit_multiple_model(self):
- # WARNING: set orca off to prevent unwanted redistribution
- with OptimizerControl(False):
- self.start_training_time = datetime.datetime.now()
- self.metrics_elapsed_start_time = time.time()
- self.train_multiple_model()
- self.end_training_time = datetime.datetime.now()
-
- def cleanup_for_warm_start(self):
- """
- 1. drop original model table
- 2. rename temp to original
- :return:
- """
- drop_query = "DROP TABLE IF EXISTS {}".format(
- self.original_model_output_table)
- plpy.execute(drop_query)
- rename_table(self.schema_madlib, self.model_output_table,
- self.original_model_output_table)
-
- def train_multiple_model(self):
- total_msts = len(self.msts_for_schedule)
- for iter in range(1, self.num_iterations+1):
- for mst_idx in range(total_msts):
- mst_row = [self.grand_schedule[dist_key][mst_idx]
- for dist_key in self.dist_keys]
- self.create_mst_schedule_table(mst_row)
- self.is_final_training_call = (iter == self.num_iterations and mst_idx == total_msts-1)
- if mst_idx == 0:
- start_iteration = time.time()
- self.run_training(mst_idx)
- if mst_idx == (total_msts - 1):
- end_iteration = time.time()
- self.info_str = "\tTime for training in iteration " \
- "{0}: {1} sec\n".format(iter,
- end_iteration -
- start_iteration)
- if should_compute_metrics_this_iter(iter,
- self.metrics_compute_frequency,
- self.num_iterations):
- self.metrics_iters.append(iter)
- self.info_str += "\tTraining set after iteration {0}:".format(iter)
- self.evaluate_model(iter, self.source_table, True)
- if self.validation_table:
- self.evaluate_model(iter, self.validation_table, False)
- plpy.info("\n"+self.info_str)
-
- def evaluate_model(self, epoch, table, is_train):
- if is_train:
- mst_metric_eval_time = self.train_mst_metric_eval_time
- mst_loss = self.train_mst_loss
- mst_metric = self.train_mst_metric
- seg_ids = self.dist_key_mapping
- images_per_seg = self.images_per_seg_train
- else:
- mst_metric_eval_time = self.valid_mst_metric_eval_time
- mst_loss = self.valid_mst_loss
- mst_metric = self.valid_mst_metric
- seg_ids = self.dist_key_mapping_valid
- images_per_seg = self.images_per_seg_valid
- self.info_str += "\n\tValidation set after iteration {0}:".format(epoch)
- for mst in self.msts:
- weights = query_weights(self.model_output_table, self.model_weights_col,
- self.mst_key_col, mst[self.mst_key_col])
- model_arch, _ = get_model_arch_weights(self.model_arch_table, mst[self.model_id_col])
- _, metric, loss = compute_loss_and_metrics(
- self.schema_madlib, table, "$madlib${0}$madlib$".format(
- mst[self.compile_params_col]),
- model_arch,
- weights,
- self.use_gpus,
- self.accessible_gpus_for_seg,
- seg_ids,
- images_per_seg,
- [], [], epoch, True)
- mst_metric_eval_time[mst[self.mst_key_col]] \
- .append(time.time() - self.metrics_elapsed_start_time)
- mst_loss[mst[self.mst_key_col]].append(loss)
- mst_metric[mst[self.mst_key_col]].append(metric)
- self.info_str += "\n\tmst_key={0}: metric={1}, loss={2}".format(mst[self.mst_key_col], metric, loss)
-
- def generate_schedule(self, msts):
- """ Generate the schedule for models hopping to segments """
- grand_schedule = {}
- for index, dist_key in enumerate(self.dist_keys):
- grand_schedule[dist_key] = rotate(msts, index)
- return grand_schedule
-
- def create_mst_schedule_table(self, mst_row):
- mst_temp_query = """
- CREATE {self.unlogged_table} TABLE {self.mst_current_schedule_tbl}
- ({self.model_id_col} INTEGER,
- {self.compile_params_col} VARCHAR,
- {self.fit_params_col} VARCHAR,
- {dist_key_col} INTEGER,
- {self.mst_key_col} INTEGER)
- """.format(dist_key_col=dist_key_col, **locals())
- plpy.execute(mst_temp_query)
- for mst, dist_key in zip(mst_row, self.dist_keys):
- if mst:
- model_id = mst[self.model_id_col]
- compile_params = mst[self.compile_params_col]
- fit_params = mst[self.fit_params_col]
- mst_key = mst[self.mst_key_col]
- else:
- model_id = "NULL"
- compile_params = "NULL"
- fit_params = "NULL"
- mst_key = "NULL"
- mst_insert_query = """
- INSERT INTO {self.mst_current_schedule_tbl}
- VALUES ({model_id},
- $madlib${compile_params}$madlib$,
- $madlib${fit_params}$madlib$,
- {dist_key},
- {mst_key})
- """.format(**locals())
- plpy.execute(mst_insert_query)
-
-
- def create_model_output_table(self):
- output_table_create_query = """
- CREATE TABLE {self.model_output_table}
- ({self.mst_key_col} INTEGER PRIMARY KEY,
- {self.model_weights_col} BYTEA,
- {self.model_arch_col} JSON)
- """.format(self=self)
- plpy.execute(output_table_create_query)
- self.initialize_model_output_and_info()
-
- def create_model_output_table_warm_start(self):
- """
- For warm start, we need to copy the model output table to a temp table
- because we call truncate on the model output table while training.
- If the query gets aborted, we need to make sure that the user passed
- model output table can be recovered.
- """
- plpy.execute("""
- CREATE TABLE {self.model_output_table} (
- LIKE {self.original_model_output_table} INCLUDING indexes);
- """.format(self=self))
-
- plpy.execute("""INSERT INTO {self.model_output_table}
- SELECT * FROM {self.original_model_output_table};
- """.format(self=self))
-
- plpy.execute(""" DELETE FROM {self.model_output_table}
- WHERE {self.mst_key_col} NOT IN (
- SELECT {self.mst_key_col} FROM {self.model_selection_table})
- """.format(self=self))
- self.warm_start_msts = plpy.execute(
- """ SELECT array_agg({0}) AS a FROM {1}
- """.format(self.mst_key_col, self.model_output_table))[0]['a']
- plpy.execute("DROP TABLE {0}".format(self.model_info_table))
- self.initialize_model_output_and_info()
-
- def initialize_model_output_and_info(self):
- info_table_create_query = """
- CREATE TABLE {self.model_info_table}
- ({self.mst_key_col} INTEGER PRIMARY KEY,
- {self.model_id_col} INTEGER,
- {self.compile_params_col} TEXT,
- {self.fit_params_col} TEXT,
- model_type TEXT,
- model_size DOUBLE PRECISION,
- metrics_elapsed_time DOUBLE PRECISION[],
- metrics_type TEXT[],
- training_metrics_final DOUBLE PRECISION,
- training_loss_final DOUBLE PRECISION,
- training_metrics DOUBLE PRECISION[],
- training_loss DOUBLE PRECISION[],
- validation_metrics_final DOUBLE PRECISION,
- validation_loss_final DOUBLE PRECISION,
- validation_metrics DOUBLE PRECISION[],
- validation_loss DOUBLE PRECISION[])
- """.format(self=self)
-
- plpy.execute(info_table_create_query)
- for mst in self.msts:
- model_arch, model_weights = get_model_arch_weights(self.model_arch_table,
- mst[self.model_id_col])
-
-
- # If warm start is enabled, weights from transfer learning cannot be
- # used, even if a particular model doesn't have warm start weights.
- if self.warm_start:
- model_weights = None
- mst_filter = """
- WHERE {mst_col}={mst_key}
- """.format(
- mst_col=self.mst_key_col,
- mst_key=mst['mst_key']
- )
-
- else:
- mst_filter = ''
-
- serialized_weights = get_initial_weights(self.model_output_table,
- model_arch,
- model_weights,
- mst['mst_key'] in self.warm_start_msts,
- self.use_gpus,
- self.accessible_gpus_for_seg,
- mst_filter
- )
- model_size = sys.getsizeof(serialized_weights) / 1024.0
-
- metrics_list = get_metrics_from_compile_param(
- mst[self.compile_params_col])
- is_metrics_specified = True if metrics_list else False
- metrics_type = 'ARRAY{0}'.format(
- metrics_list) if is_metrics_specified else 'NULL'
-
- info_table_insert_query = """
- INSERT INTO {self.model_info_table}({self.mst_key_col},
- {self.model_id_col}, {self.compile_params_col},
- {self.fit_params_col}, model_type, model_size,
- metrics_type)
- VALUES ({mst_key_val}, {model_id},
- $madlib${compile_params}$madlib$,
- $madlib${fit_params}$madlib$, '{model_type}',
- {model_size}, {metrics_type})
- """.format(self=self,
- mst_key_val=mst[self.mst_key_col],
- model_id=mst[self.model_id_col],
- compile_params=mst[self.compile_params_col],
- fit_params=mst[self.fit_params_col],
- model_type='madlib_keras',
- model_size=model_size,
- metrics_type=metrics_type)
- plpy.execute(info_table_insert_query)
-
- if not mst['mst_key'] in self.warm_start_msts:
- output_table_insert_query = """
- INSERT INTO {self.model_output_table}(
- {self.mst_key_col}, {self.model_weights_col},
- {self.model_arch_col})
- VALUES ({mst_key}, $1, $2)
- """.format(self=self,
- mst_key=mst[self.mst_key_col])
- output_table_insert_query_prepared = plpy.prepare(
- output_table_insert_query, ["bytea", "json"])
- plpy.execute(output_table_insert_query_prepared, [
- serialized_weights, model_arch])
-
- def create_model_summary_table(self):
- if self.warm_start:
- plpy.execute("DROP TABLE {0}".format(self.model_summary_table))
- src_summary_dict = get_source_summary_table_dict(self.fit_validator_train)
- class_values = src_summary_dict['class_values']
- class_values_type = src_summary_dict['class_values_type']
- dep_vartype = src_summary_dict['dep_vartype']
- dependent_varname = \
- src_summary_dict['dependent_varname_in_source_table']
- independent_varname = \
- src_summary_dict['independent_varname_in_source_table']
- norm_const = src_summary_dict['norm_const']
- self.validation_table = 'NULL' if self.validation_table is None \
- else '$MAD${0}$MAD$'.format(self.validation_table)
- if class_values is None:
- num_classes = 'NULL'
- else:
- num_classes = len(class_values)
- name = 'NULL' if self.name is None else '$MAD${0}$MAD$'.format(self.name)
- descr = 'NULL' if self.description is None else '$MAD${0}$MAD$'.format(self.description)
- metrics_iters = self.metrics_iters if self.metrics_iters else 'NULL'
- class_values_colname = CLASS_VALUES_COLNAME
- dependent_vartype_colname = DEPENDENT_VARTYPE_COLNAME
- normalizing_const_colname = NORMALIZING_CONST_COLNAME
- float32_sql_type = FLOAT32_SQL_TYPE
- create_query = plpy.prepare("""
- CREATE TABLE {self.model_summary_table} AS
- SELECT
- $MAD${self.source_table}$MAD$::TEXT AS source_table,
- {self.validation_table}::TEXT AS validation_table,
- $MAD${self.model_output_table}$MAD$::TEXT AS model,
- $MAD${self.model_info_table}$MAD$::TEXT AS model_info,
- $MAD${dependent_varname}$MAD$::TEXT AS dependent_varname,
- $MAD${independent_varname}$MAD$::TEXT AS independent_varname,
- $MAD${self.model_arch_table}$MAD$::TEXT AS model_arch_table,
- {self.num_iterations}::INTEGER AS num_iterations,
- {self.metrics_compute_frequency}::INTEGER AS metrics_compute_frequency,
- {self.warm_start} AS warm_start,
- {name}::TEXT AS name,
- {descr}::TEXT AS description,
- '{self.start_training_time}'::TIMESTAMP AS start_training_time,
- '{self.end_training_time}'::TIMESTAMP AS end_training_time,
- '{self.version}'::TEXT AS madlib_version,
- {num_classes}::INTEGER AS num_classes,
- $1 AS {class_values_colname},
- $MAD${dep_vartype}$MAD$::TEXT AS {dependent_vartype_colname},
- {norm_const}::{float32_sql_type} AS {normalizing_const_colname},
- ARRAY{metrics_iters}::INTEGER[] AS metrics_iters
- """.format(**locals()), [class_values_type])
- plpy.execute(create_query, [class_values])
-
- def update_info_table(self, mst, is_train):
- mst_key = mst[self.mst_key_col]
- metrics, metrics_final, metrics_elapsed_time = \
- "NULL", "NULL", "NULL"
- if is_train:
- mst_metric = self.train_mst_metric
- mst_metric_eval_time = self.train_mst_metric_eval_time
- mst_loss = self.train_mst_loss
- else:
- mst_metric = self.valid_mst_metric
- mst_metric_eval_time = self.valid_mst_metric_eval_time
- mst_loss = self.valid_mst_loss
-
- if mst_key in mst_metric:
- metrics = mst_metric[mst_key]
- metrics_final = metrics[-1]
- metrics_elapsed_time = mst_metric_eval_time[mst_key]
- metrics = "ARRAY{}".format(metrics)
- metrics_elapsed_time = "ARRAY{}".format(metrics_elapsed_time)
- loss = mst_loss[mst_key]
- loss_final = loss[-1]
- loss = "ARRAY{}".format(loss)
- if is_train:
- update_query = """
- UPDATE {self.model_info_table} SET
- training_metrics_final = {metrics_final},
- training_loss_final = {loss_final},
- metrics_elapsed_time = {metrics_elapsed_time},
- training_metrics = {metrics},
- training_loss = {loss}
- WHERE {self.mst_key_col} = {mst_key}
- """.format(**locals())
- else:
- update_query = """
- UPDATE {self.model_info_table} SET
- validation_metrics_final = {metrics_final},
- validation_loss_final = {loss_final},
- metrics_elapsed_time = {metrics_elapsed_time},
- validation_metrics = {metrics},
- validation_loss = {loss}
- WHERE {self.mst_key_col} = {mst_key}
- """.format(**locals())
- plpy.execute(update_query)
-
- def insert_info_table(self):
- for mst in self.msts:
- self.update_info_table(mst, True)
- if self.validation_table:
- self.update_info_table(mst, False)
-
- def run_training(self, mst_idx):
- # NOTE: In the DL module, we want to avoid CREATING TEMP tables
- # (creates a slice which stays until the session is disconnected)
- # or minimize writing queries that generate plans with Motions (creating
- # multiple slices on segments).
- # This is mainly to avoid any GPU memory allocation failures. Since GPU
- # memory allocation is tied to the process where it is initialized, failures
- # may occur when a newly created slice(process) tries allocating GPU memory
- # which is already allocated by a previously created slice(process).
- # Therefore we want to have queries that do not add motions and all the
- # sub-queries running Keras/tensorflow operations reuse the same slice(process)
- # that was used for initializing GPU memory.
- use_gpus = self.use_gpus if self.use_gpus else False
- mst_weights_query = """
- CREATE {self.unlogged_table} TABLE {self.mst_weights_tbl} AS
- SELECT mst_tbl.*, wgh_tbl.{self.model_weights_col},
- model_arch_tbl.{self.model_arch_col}
- FROM
- {self.mst_current_schedule_tbl} mst_tbl
- LEFT JOIN {self.model_output_table} wgh_tbl
- ON mst_tbl.{self.mst_key_col} = wgh_tbl.{self.mst_key_col}
- LEFT JOIN {self.model_arch_table} model_arch_tbl
- ON mst_tbl.{self.model_id_col} = model_arch_tbl.{self.model_id_col}
- DISTRIBUTED BY ({dist_key_col})
- """.format(dist_key_col=dist_key_col,
- **locals())
- plpy.execute(mst_weights_query)
- use_gpus = self.use_gpus if self.use_gpus else False
- uda_query = """
- CREATE {self.unlogged_table} TABLE {self.weights_to_update_tbl} AS
- SELECT {self.schema_madlib}.fit_step_multiple_model({mb_dep_var_col},
- {mb_indep_var_col},
- {self.dep_shape_col},
- {self.ind_shape_col},
- {self.mst_weights_tbl}.{self.model_arch_col}::TEXT,
- {self.mst_weights_tbl}.{self.compile_params_col}::TEXT,
- {self.mst_weights_tbl}.{self.fit_params_col}::TEXT,
- src.{dist_key_col},
- ARRAY{self.dist_key_mapping},
- src.{self.gp_segment_id_col},
- {self.segments_per_host},
- ARRAY{self.images_per_seg_train},
- {use_gpus}::BOOLEAN,
- ARRAY{self.accessible_gpus_for_seg},
- {self.mst_weights_tbl}.{self.model_weights_col}::BYTEA,
- {is_final_iteration}::BOOLEAN
- )::BYTEA AS {self.model_weights_col},
- {self.mst_weights_tbl}.{self.mst_key_col} AS {self.mst_key_col}
- ,src.{dist_key_col} AS {dist_key_col}
- FROM {self.source_table} src JOIN {self.mst_weights_tbl}
- USING ({dist_key_col})
- WHERE {self.mst_weights_tbl}.{self.mst_key_col} IS NOT NULL
- GROUP BY src.{dist_key_col}, {self.mst_weights_tbl}.{self.mst_key_col}
- DISTRIBUTED BY({dist_key_col})
- """.format(mb_dep_var_col=mb_dep_var_col,
- mb_indep_var_col=mb_indep_var_col,
- is_final_iteration=True,
- dist_key_col=dist_key_col,
- use_gpus=use_gpus,
- self=self
- )
- plpy.execute(uda_query)
-
- update_query = """
- UPDATE {self.model_output_table}
- SET {self.model_weights_col} = {self.weights_to_update_tbl}.{self.model_weights_col}
- FROM {self.weights_to_update_tbl}
- WHERE {self.model_output_table}.{self.mst_key_col} = {self.weights_to_update_tbl}.{self.mst_key_col}
- """.format(self=self)
- plpy.execute(update_query)
-
- self.truncate_and_drop_tables()
-
- def truncate_and_drop_tables(self):
- """
- Context: UPDATE statements in postgres are not in-place replacements but
- the row to be updated is marked for deletion(note that the disk space for
- this row doesn't get released until vaccuum is called) and a new row in
- inserted.
-
- This function will clear out the disk space used by the model_output_table
- and also drop all the other intermediate tables.
- If available, set the `` guc so that the truncate command can release the
- disk space. The disk space will be released immediately and hence the
- model_output table won't grow in size with each UPDATE statement.
-
- Without this guc, the disk space won't be released and each
- call to the UPDATE statement will keep adding to the disk space. The disk
- space will only be released when the query is completed.
-
- The guc can cause data loss if not used properly. Since truncate will
- actually clear the disk space immediately, there is no way to recover to
- the state before truncate was called on that table. So this guc should only
- be set for intermediate tables and never for tables created outside the
- scope of the fit_multiple udf.
-
- Workflow
- 1. Create temp table from model table (including the indexes)
- 2. truncate the model table to release disk space
- 3. rename temp table to model table so that it can be reused for the next
- hop
- :return:
- """
-
- with SetGUC("dev_opt_unsafe_truncate_in_subtransaction", "on"):
- temp_model_table = unique_string('updated_model')
- unlogged_table = self.unlogged_table if not self.is_final_training_call else ''
- plpy.execute("""
- CREATE {unlogged_table} TABLE {temp_model_table} ( LIKE {self.model_output_table}
- INCLUDING indexes);""".format(temp_model_table=temp_model_table,
- unlogged_table=unlogged_table,
- self=self))
- plpy.execute("""
- INSERT INTO {temp_model_table} SELECT * FROM {self.model_output_table};
- TRUNCATE TABLE {self.model_output_table};
- DROP TABLE {self.model_output_table};
- """.format(temp_model_table=temp_model_table, self=self))
- rename_table(self.schema_madlib, temp_model_table,
- self.model_output_table)
- plpy.execute("""
- TRUNCATE TABLE {self.mst_weights_tbl}, {self.mst_current_schedule_tbl},
- {self.weights_to_update_tbl};
- DROP TABLE IF EXISTS {self.mst_weights_tbl}, {self.mst_current_schedule_tbl},
- {self.weights_to_update_tbl};""".format(self=self))
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
deleted file mode 100644
index 8d68385..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
+++ /dev/null
@@ -1,1555 +0,0 @@
-/* ----------------------------------------------------------------------- *//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *
- * @file madlib_keras_model_selection.sql_in
- *
- * @brief SQL functions for model hopper distributed training
- * @date August 2019
- *
- *
- *//* ----------------------------------------------------------------------- */
-
-m4_include(`SQLCommon.m4')
-
-/**
-@addtogroup grp_keras_run_model_selection
-
-@brief Explore network architectures and hyperparameters by training many models a time.
-
-<div class="toc"><b>Contents</b><ul>
-<li class="level1"><a href="#keras_fit">Fit</a></li>
-<li class="level1"><a href="#keras_evaluate">Evaluate</a></li>
-<li class="level1"><a href="#keras_predict">Predict</a></li>
-<li class="level1"><a href="#example">Examples</a></li>
-<li class="level1"><a href="#notes">Notes</a></li>
-<li class="level1"><a href="#background">Technical Background</a></li>
-<li class="level1"><a href="#literature">Literature</a></li>
-<li class="level1"><a href="#related">Related Topics</a></li>
-</ul></div>
-
-\warning <em> This MADlib method is still in early stage development.
-Interface and implementation are subject to change. </em>
-
-This module allows you to explore network architectures and
-hyperparameters by training many models a time across the
-database cluster. The aim is to support efficient empirical comparison of multiple
-training configurations. This process is called model selection,
-and the implementation here is based on a parallel execution strategy
-called model hopper parallelism (MOP) [1,2].
-
-Models are designed in Keras [3], which is a high-level neural
-network API written in Python. It can run
-on top of different backends and the one that is currently
-supported by MADlib is TensorFlow [4].
-
-The main use case is image classification
-using sequential models, which are made up of a
-linear stack of layers. This includes multilayer perceptrons (MLPs)
-and convolutional neural networks (CNNs). Regression is not
-currently supported.
-
-Before doing model selection in MADlib you will need to run
-the mini-batch preprocessor, and create a table with the various models
-and hyperparameters to try.
-
-You can mini-batch the training and evaluation datasets by using the
-<a href="group__grp__input__preprocessor__dl.html">Preprocessor
-for Images</a> which is a utility that prepares image data for
-use by models that support mini-batch as an optimization option.
-This is a one-time operation and you would only
-need to re-run the preprocessor if your input data has changed.
-The advantage of using mini-batching is that it
-can perform better than stochastic gradient descent
-because it uses more than one training example at a time,
-typically resulting faster and smoother convergence [5].
-The input preprocessor also sets the distribution rules
-for the training data. For example, you may only want
-to train models on segments that reside on hosts that are GPU enabled.
-
-You can set up the models and hyperparameters to try with the
-<a href="group__grp__keras__setup__model__selection.html">Setup
-Model Selection</a> utility to define the unique combinations
-of model architectures, compile and fit parameters.
-
-@note If 'madlib_keras_fit_multiple_model()' is running on GPDB 5 and some versions
-of GPDB 6, the database will
-keep adding to the disk space (in proportion to model size) and will only
-release the disk space once the fit multiple query has completed execution.
-This is not the case for GPDB 6.5.0+ where disk space is released during the
-fit multiple query.
-
-@note CUDA GPU memory cannot be released until the process holding it is terminated.
-When a MADlib deep learning function is called with GPUs, Greenplum internally
-creates a process (called a slice) which calls TensorFlow to do the computation.
-This process holds the GPU memory until one of the following two things happen:
-query finishes and user logs out of the Postgres client/session; or,
-query finishes and user waits for the timeout set by gp_vmem_idle_resource_timeout.
-The default value for this timeout is 18 sec [8]. So the recommendation is:
-log out/reconnect to the session after every GPU query; or
-wait for gp_vmem_idle_resource_timeout before you run another GPU query (you can
-also set it to a lower value).
-
-@anchor keras_fit
-@par Fit
-The fit (training) function has the following format:
-
-<pre class="syntax">
-madlib_keras_fit_multiple_model(
- source_table,
- model_output_table,
- model_selection_table,
- num_iterations,
- use_gpus,
- validation_table,
- metrics_compute_frequency,
- warm_start,
- name,
- description
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
- <dt>source_table</dt>
- <dd>TEXT. Name of the table containing the training data.
- This is the name of the output
- table from the image preprocessor. Independent
- and dependent variables are specified in the preprocessor
- step which is why you do not need to explictly state
- them here as part of the fit function.</dd>
-
- <dt>model_output_table</dt>
- <dd>TEXT. Name of the output table containing the
- multiple models created.
- @note pg_temp is not allowed as an output table schema for fit multiple.
- Details of output tables are shown below.
- </dd>
-
- <dt>model_selection_table</dt>
- <dd>TEXT. Name of the table containing model selection parameters to be tried.
- Here we mean both hyper-parameter tuning and model architecture search.
- </dd>
-
- <DT>num_iterations</DT>
- <DD>INTEGER. Number of iterations to train.
-
- @note
- This parameter is different than the number of passes over the dataset,
- which is commonly referred to as the number of epochs. Since MADlib operates
- in a distributed system, the number of
- epochs is actually equal to this parameter 'num_iterations' X 'epochs' as
- specified in the Keras fit parameter.
- </DD>
-
- <DT>use_gpus (optional)</DT>
- <DD>BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs
- are to be used for training the neural network. Set to TRUE to use GPUs.
-
- @note
- This parameter must not conflict with how the distribution rules are set in
- the preprocessor function. For example, if you set a distribution rule to use
- certain segments on hosts that do not have GPUs attached, you will get an error
- if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues
- when segments share GPU resources.
- For example, if you have 1 GPU per segment host and your cluster has 4
- segments per segment host, it means that all 4
- segments will share the same
- GPU on each host. The current recommended
- configuration is 1 GPU per segment.
- </DD>
-
- <dt>validation_table (optional)</dt>
- <dd>TEXT, default: none. Name of the table containing
- the validation dataset.
- Note that the validation dataset must be preprocessed
- in the same way as the training dataset, so this
- is the name of the output
- table from running the image preprocessor on the validation dataset.
- Using a validation dataset can mean a
- longer training time, depending on its size.
- This can be controlled using the 'metrics_compute_frequency'
- parameter described below.</dd>
-
- <DT>metrics_compute_frequency (optional)</DT>
- <DD>INTEGER, default: once at the end of training
- after 'num_iterations'. Frequency to compute per-iteration
- metrics for the training dataset and validation dataset
- (if specified). There can be considerable cost to
- computing metrics every iteration, especially if the
- training dataset is large. This parameter is a way of
- controlling the frequency of those computations.
- For example, if you specify 5, then metrics will be computed
- every 5 iterations as well as at the end of training
- after 'num_iterations'. If you use the default,
- metrics will be computed only
- once after 'num_iterations' have completed.
- </DD>
-
- <DT>warm_start (optional)</DT>
- <DD>BOOLEAN, default: FALSE.
- Initalize weights with the coefficients
- from the last call to the fit
- function. If set to TRUE, weights will be
- initialized from the model table
- generated by the previous training run.
-
- @note
- The warm start feature works based on the name of the
- model output table from a previous training run.
- When using warm start, do not drop the model output table
- or the model output summary table
- before calling the fit function, since these are needed to obtain the
- weights from the previous run.
- If you are not using warm start, the model output table
- and the model output table summary must be dropped in
- the usual way before calling the training function.
- </DD>
-
- <DT>name (optional)</DT>
- <DD>TEXT, default: NULL.
- Free text string to identify a name, if desired.
- </DD>
-
- <DT>description (optional)</DT>
- <DD>TEXT, default: NULL.
- Free text string to provide a description, if desired.
- </DD>
-</dl>
-
-<b>Output tables</b>
-<br>
- The model output table produced by fit contains the following columns.
- There is one row per model as per the rows in the 'model_selection_table':
- <table class="output">
- <tr>
- <th>mst_key</th>
- <td>INTEGER. ID that defines a unique tuple for model architecture-compile parameters-fit parameters,
- as defined in the 'model_selection_table'.</td>
- </tr>
- <tr>
- <th>model_weights</th>
- <td>BYTEA8. Byte array containing the weights of the neural net.</td>
- </tr>
- <tr>
- <th>model_arch</th>
- <td>TEXT. A JSON representation of the model architecture
- used in training.</td>
- </tr>
- </table>
-
- An info table named \<model_output_table\>_info is also created, which has the following columns.
- There is one row per model as per the rows in the 'model_selection_table':
- <table class="output">
- <tr>
- <th>mst_key</th>
- <td>INTEGER. ID that defines a unique tuple for model architecture-compile parameters-fit parameters,
- as defined in the 'model_selection_table'.</td>
- </tr>
- <tr>
- <th>model_id</th>
- <td>INTEGER. ID that defines model in the 'model_arch_table'.</td>
- </tr>
- <tr>
- <th>compile_params</th>
- <td>Compile parameters passed to Keras.</td>
- </tr>
- <tr>
- <th>fit_params</th>
- <td>Fit parameters passed to Keras.</td>
- </tr>
- <tr>
- <th>model_type</th>
- <td>General identifier for type of model trained.
- Currently says 'madlib_keras'.</td>
- </tr>
- <tr>
- <th>model_size</th>
- <td>Size of the model in KB. Models are stored in
- 'bytea' data format which is used for binary strings
- in PostgreSQL type databases.</td>
- </tr>
- <tr>
- <th>metrics_elapsed_time</th>
- <td> Array of elapsed time for metric computations as
- per the 'metrics_compute_frequency' parameter.
- Useful for drawing a curve showing loss, accuracy or
- other metrics as a function of time.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of elapsed time for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>metrics_type</th>
- <td>Metric specified in the 'compile_params'.</td>
- </tr>
- <tr>
- <th>training_metrics_final</th>
- <td>Final value of the training
- metric after all iterations have completed.
- The metric reported is the one
- specified in the 'metrics_type' parameter.</td>
- </tr>
- <tr>
- <th>training_loss_final</th>
- <td>Final value of the training loss after all
- iterations have completed.</td>
- </tr>
- <tr>
- <th>training_metrics</th>
- <td>Array of training metrics as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of metrics for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>training_loss</th>
- <td>Array of training losses as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of losses for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>validation_metrics_final</th>
- <td>Final value of the validation
- metric after all iterations have completed.
- The metric reported is the one
- specified in the 'metrics_type' parameter.</td>
- </tr>
- <tr>
- <th>validation_loss_final</th>
- <td>Final value of the validation loss after all
- iterations have completed.</td>
- </tr>
- <tr>
- <th>validation_metrics</th>
- <td>Array of validation metrics as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of metrics for every 5th
- iteration, plus the last iteration.</td>
- </tr>
- <tr>
- <th>validation_loss</th>
- <td>Array of validation losses as
- per the 'metrics_compute_frequency' parameter.
- For example, if 'metrics_compute_frequency=5'
- this would be an array of losses for every 5th
- iteration, plus the last iteration.</td>
- </tr>
-
- </table>
-
- A summary table named \<model\>_summary is also created, which has the following columns:
- <table class="output">
- <tr>
- <th>source_table</th>
- <td>Source table used for training.</td>
- </tr>
- <tr>
- <th>validation_table</th>
- <td>Name of the table containing
- the validation dataset (if specified).</td>
- </tr>
- <tr>
- <th>model</th>
- <td>Name of the output table containing
- the model for each model selection tuple.</td>
- </tr>
- <tr>
- <th>model_info</th>
- <td>Name of the output table containing
- the model performance and other info for
- each model selection tuple.</td>
- </tr>
- <tr>
- <th>dependent_varname</th>
- <td>Dependent variable column from the original
- source table in the image preprocessing step.</td>
- </tr>
- <tr>
- <th>independent_varname</th>
- <td>Independent variables column from the original
- source table in the image preprocessing step.</td>
- </tr>
- <tr>
- <th>model_arch_table</th>
- <td>Name of the table containing
- the model architecture and (optionally) the
- initial model weights.</td>
- </tr>
- <tr>
- <th>num_iterations</th>
- <td>Number of iterations of training completed.</td>
- </tr>
- <tr>
- <th>metrics_compute_frequency</th>
- <td>Frequency that per-iteration metrics are computed
- for the training dataset and validation
- datasets.</td>
- </tr>
- <tr>
- <th>warm_start</th>
- <td>Indicates whether warm start used or not.</td>
- </tr>
- <tr>
- <th>name</th>
- <td>Name of the training run (free text).</td>
- </tr>
- <tr>
- <th>description</th>
- <td>Description of the training run (free text).</td>
- </tr>
- <tr>
- <th>start_training_time</th>
- <td>Timestamp for start of training.</td>
- </tr>
- <tr>
- <th>end_training_time</th>
- <td>Timestamp for end of training.</td>
- </tr>
- <tr>
- <th>madlib_version</th>
- <td>Version of MADlib used.</td>
- </tr>
- <tr>
- <th>num_classes</th>
- <td>Count of distinct classes values used.</td>
- </tr>
- <tr>
- <th>class_values</th>
- <td>Array of actual class values used.</td>
- </tr>
- <tr>
- <th>dependent_vartype</th>
- <td>Data type of the dependent variable.</td>
- </tr>
- <tr>
- <th>normalizing_constant</th>
- <td>Normalizing constant used from the
- image preprocessing step.</td>
- </tr>
- <tr>
- <th>metrics_iters</th>
- <td>Array indicating the iterations for which
- metrics are calculated, as derived from the
- parameters 'num_iterations' and 'metrics_compute_frequency'.
- For example, if 'num_iterations=5'
- and 'metrics_compute_frequency=2', then 'metrics_iters' value
- would be {2,4,5} indicating that metrics were computed
- at iterations 2, 4 and 5 (at the end).
- If 'num_iterations=5'
- and 'metrics_compute_frequency=1', then 'metrics_iters' value
- would be {1,2,3,4,5} indicating that metrics were computed
- at every iteration.</td>
- </tr>
- </table>
-
-@anchor keras_evaluate
-@par Evaluate
-The evaluation function has the following format:
-
-<pre class="syntax">
-madlib_keras_evaluate(
- model_table,
- test_table,
- output_table,
- use_gpus
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
-
-<DT>model_table</DT>
- <DD>TEXT. Name of the table containing the model
- to use for validation.
- </DD>
-
- <DT>test_table</DT>
- <dd>TEXT. Name of the table containing the evaluation dataset.
- Note that test/validation data must be preprocessed in the same
- way as the training dataset, so
- this is the name of the output
- table from the image preprocessor. Independent
- and dependent variables are specified in the preprocessor
- step which is why you do not need to explictly state
- them here as part of the fit function.</dd>
-
- <DT>output_table</DT>
- <DD>TEXT. Name of table that validation output will be
- written to. Table contains:</DD>
- <table class="output">
- <tr>
- <th>loss</th>
- <td>Loss value on evaluation dataset.</td>
- </tr>
- <tr>
- <th>metric</th>
- <td>Metric value on evaluation dataset, where 'metrics_type'
- below identifies the type of metric.</td>
- </tr>
- <tr>
- <th>metrics_type</th>
- <td>Type of metric used that was used in the training step.</td>
- </tr>
-
- <DT>use_gpus (optional)</DT>
- <DD>BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs
- are to be used for training the neural network. Set to TRUE to use GPUs.
-
- @note
- This parameter must not conflict with how the distribution rules are set in
- the preprocessor function. For example, if you set a distribution rule to use
- certain segments on hosts that do not have GPUs attached, you will get an error
- if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues
- when segments share GPU resources.
- For example, if you have 1 GPU per segment host and your cluster has 4
- segments per segment host, it means that all 4
- segments will share the same
- GPU on each host. The current recommended
- configuration is 1 GPU per segment.
- </DD>
-</DL>
-
-@anchor keras_predict
-@par Predict
-The prediction function has the following format:
-<pre class="syntax">
-madlib_keras_predict(
- model_table,
- test_table,
- id_col,
- independent_varname,
- output_table,
- pred_type,
- use_gpus
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
-
-<DT>model_table</DT>
- <DD>TEXT. Name of the table containing the model
- to use for prediction.
- </DD>
-
- <DT>test_table</DT>
- <DD>TEXT. Name of the table containing the dataset to
- predict on. Note that test data is not preprocessed (unlike
- fit and evaluate) so put one test image per row for prediction.
- Also see the comment below for the 'independent_varname' parameter
- regarding normalization.
-
- </DD>
-
- <DT>id_col</DT>
- <DD>TEXT. Name of the id column in the test data table.
- </DD>
-
- <DT>independent_varname</DT>
- <DD>TEXT. Column with independent variables in the test table.
- If a 'normalizing_const' is specified when preprocessing the
- training dataset, this same normalization will be applied to
- the independent variables used in predict.
- </DD>
-
- <DT>output_table</DT>
- <DD>TEXT. Name of the table that prediction output will be
- written to. Table contains:</DD>
- <table class="output">
- <tr>
- <th>id</th>
- <td>Gives the 'id' for each prediction, corresponding to each row from the test_table.</td>
- </tr>
- <tr>
- <th>estimated_COL_NAME</th>
- <td>
- (For pred_type='response') The estimated class
- for classification, where
- COL_NAME is the name of the column to be
- predicted from test data.
- </td>
- </tr>
- <tr>
- <th>prob_CLASS</th>
- <td>
- (For pred_type='prob' for classification) The
- probability of a given class.
- There will be one column for each class
- in the training data.
- </td>
- </tr>
-
- <DT>pred_type (optional)</DT>
- <DD>TEXT, default: 'response'. The type of output
- desired, where 'response' gives the actual prediction
- and 'prob' gives the probability value for each class.
- </DD>
-
- <DT>use_gpus (optional)</DT>
- <DD>BOOLEAN, default: FALSE (i.e., CPU). Determines
- whether GPUs are to be used for prediction/inference.
- Set to TRUE to use GPUs.
-
- @note
- The prediction function uses the whole cluster. If you are using GPUs, it
- requires that GPUs are attached to all hosts, and that there are the same number
- of GPUs on each host (homogeneous cluster). This is different from the fit()
- and evaluate() functions that support GPUs on only some of the hosts (heterogeneous cluster).
- Therefore, if you have GPUs only on some of the hosts, or an uneven numbers of GPUs per host, then
- set this parameter to FALSE to use CPUs.
- </DD>
-</DL>
-
-@anchor example
-@par Examples
-
-@note
-Deep learning works best on very large datasets,
-but that is not convenient for a quick introduction
-to the syntax. So in this example we use an MLP on the well
-known iris data set from https://archive.ics.uci.edu/ml/datasets/iris.
-For more realistic examples with images please refer
-to the deep learning notebooks
-at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts.
-
-<h4>Classification</h4>
-
--# Create an input data set.
-<pre class="example">
-DROP TABLE IF EXISTS iris_data;
-CREATE TABLE iris_data(
- id serial,
- attributes numeric[],
- class_text varchar
-);
-INSERT INTO iris_data(id, attributes, class_text) VALUES
-(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'),
-(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'),
-(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'),
-(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'),
-(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'),
-(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'),
-(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'),
-(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'),
-(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'),
-(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'),
-(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'),
-(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'),
-(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'),
-(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'),
-(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'),
-(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'),
-(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'),
-(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'),
-(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'),
-(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'),
-(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'),
-(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'),
-(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'),
-(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'),
-(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'),
-(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'),
-(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'),
-(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'),
-(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'),
-(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'),
-(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'),
-(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'),
-(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'),
-(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'),
-(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'),
-(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'),
-(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'),
-(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'),
-(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'),
-(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'),
-(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'),
-(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'),
-(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'),
-(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'),
-(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'),
-(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'),
-(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'),
-(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'),
-(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'),
-(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'),
-(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'),
-(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'),
-(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'),
-(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'),
-(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'),
-(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'),
-(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'),
-(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'),
-(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'),
-(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'),
-(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'),
-(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'),
-(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'),
-(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'),
-(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'),
-(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'),
-(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'),
-(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'),
-(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'),
-(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'),
-(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'),
-(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'),
-(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'),
-(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'),
-(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'),
-(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'),
-(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'),
-(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'),
-(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'),
-(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'),
-(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'),
-(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'),
-(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'),
-(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'),
-(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'),
-(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'),
-(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'),
-(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'),
-(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'),
-(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'),
-(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'),
-(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'),
-(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'),
-(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'),
-(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'),
-(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'),
-(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'),
-(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'),
-(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'),
-(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'),
-(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'),
-(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'),
-(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'),
-(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'),
-(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'),
-(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'),
-(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'),
-(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'),
-(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'),
-(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'),
-(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'),
-(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'),
-(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'),
-(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'),
-(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'),
-(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'),
-(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'),
-(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'),
-(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'),
-(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'),
-(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'),
-(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'),
-(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'),
-(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'),
-(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'),
-(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'),
-(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'),
-(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'),
-(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'),
-(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'),
-(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'),
-(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'),
-(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'),
-(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'),
-(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'),
-(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'),
-(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'),
-(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'),
-(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'),
-(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'),
-(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'),
-(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'),
-(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'),
-(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'),
-(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica');
-</pre>
-Create a test/validation dataset from the training data:
-<pre class="example">
-DROP TABLE IF EXISTS iris_train, iris_test;
--- Set seed so results are reproducible
-SELECT setseed(0);
-SELECT madlib.train_test_split('iris_data', -- Source table
- 'iris', -- Output table root name
- 0.8, -- Train proportion
- NULL, -- Test proportion (0.2)
- NULL, -- Strata definition
- NULL, -- Output all columns
- NULL, -- Sample without replacement
- TRUE -- Separate output tables
- );
-SELECT COUNT(*) FROM iris_train;
-</pre>
-<pre class="result">
- count
-------+
- 120
-</pre>
-
--# Call the preprocessor for deep learning. For the training dataset:
-<pre class="example">
-\\x on
-DROP TABLE IF EXISTS iris_train_packed, iris_train_packed_summary;
-SELECT madlib.training_preprocessor_dl('iris_train', -- Source table
- 'iris_train_packed', -- Output table
- 'class_text', -- Dependent variable
- 'attributes' -- Independent variable
- );
-SELECT * FROM iris_train_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------+---------------------------------------------
-source_table | iris_train
-output_table | iris_train_packed
-dependent_varname | class_text
-independent_varname | attributes
-dependent_vartype | character varying
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size | 60
-normalizing_const | 1.0
-num_classes | 3
-</pre>
-For the validation dataset:
-<pre class="example">
-DROP TABLE IF EXISTS iris_test_packed, iris_test_packed_summary;
-SELECT madlib.validation_preprocessor_dl('iris_test', -- Source table
- 'iris_test_packed', -- Output table
- 'class_text', -- Dependent variable
- 'attributes', -- Independent variable
- 'iris_train_packed' -- From training preprocessor step
- );
-SELECT * FROM iris_test_packed_summary;
-</pre>
-<pre class="result">
--[ RECORD 1 ]-------+---------------------------------------------
-source_table | iris_test
-output_table | iris_test_packed
-dependent_varname | class_text
-independent_varname | attributes
-dependent_vartype | character varying
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size | 15
-normalizing_const | 1.0
-num_classes | 3
-</pre>
-
--# Define and load model architecture. Use Keras to define
-the model architecture with 1 hidden layer:
-<pre class="example">
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model1 = Sequential()
-model1.add(Dense(10, activation='relu', input_shape=(4,)))
-model1.add(Dense(10, activation='relu'))
-model1.add(Dense(3, activation='softmax'))
-model1.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type) Output Shape Param #
-=================================================================
-dense_1 (Dense) (None, 10) 50
-_________________________________________________________________
-dense_2 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_3 (Dense) (None, 3) 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-</pre>
-Export the model to JSON:
-<pre class="example">
-model1.to_json()
-</pre>
-<pre class="result">
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-</pre>
-Define model architecture with 2 hidden layers:
-<pre class="example">
-model2 = Sequential()
-model2.add(Dense(10, activation='relu', input_shape=(4,)))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(3, activation='softmax'))
-model2.summary()
-\verbatim
-
-Layer (type) Output Shape Param #
-=================================================================
-dense_4 (Dense) (None, 10) 50
-_________________________________________________________________
-dense_5 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_6 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_7 (Dense) (None, 3) 33
-=================================================================
-Total params: 303
-Trainable params: 303
-Non-trainable params: 0
-\endverbatim
-</pre>
-Export the model to JSON:
-<pre class="example">
-model2.to_json()
-</pre>
-<pre class="result">
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-</pre>
-Load into model architecture table:
-<pre class="example">
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Sophie', -- Name
- 'MLP with 1 hidden layer' -- Descr
-);
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Maria', -- Name
- 'MLP with 2 hidden layers' -- Descr
-);
-</pre>
--# Define model selection tuples and load. Select the model(s) from the model architecture
-table that you want to run, along with the compile and fit parameters. Combinations will be
-created for the set of model selection parameters will be loaded:
-<pre class="example">
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT madlib.load_model_selection_table('model_arch_library', -- model architecture table
- 'mst_table', -- model selection table output
- ARRAY[1,2], -- model ids from model architecture table
- ARRAY[ -- compile params
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
- ],
- ARRAY[ -- fit params
- $$batch_size=4,epochs=1$$,
- $$batch_size=8,epochs=1$$
- ]
- );
-SELECT * FROM mst_table ORDER BY mst_key;
-</pre>
-<pre class="result">
- mst_key | model_id | compile_params | fit_params
----------+----------+---------------------------------------------------------------------------------+-----------------------
- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
- 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
- 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
- 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
- 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
- 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
- 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
- 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
- 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
- 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-(12 rows)
-</pre>
-This is the name of the model architecture table that corresponds to the model selection table:
-<pre class="example">
-SELECT * FROM mst_table_summary;
-</pre>
-<pre class="result">
- model_arch_table
---------------------+
- model_arch_library
-</pre>
-
--# Train multiple models.
-<pre class="example">
-DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary, iris_multi_model_info;
-SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed', -- source_table
- 'iris_multi_model', -- model_output_table
- 'mst_table', -- model_selection_table
- 10, -- num_iterations
- FALSE -- use gpus
- );
-</pre>
-View the model summary:
-<pre class="example">
-SELECT * FROM iris_multi_model_summary;
-</pre>
-<pre class="result">
-source_table | iris_train_packed
-validation_table |
-model | iris_multi_model
-model_info | iris_multi_model_info
-dependent_varname | class_text
-independent_varname | attributes
-model_arch_table | model_arch_library
-num_iterations | 10
-metrics_compute_frequency | 10
-warm_start | f
-name |
-description |
-start_training_time | 2019-12-16 18:54:33.826414
-end_training_time | 2019-12-16 18:56:19.106321
-madlib_version | 1.17.0
-num_classes | 3
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype | character varying
-normalizing_const | 1
-metrics_iters | {10}
-</pre>
-View results for each model:
-<pre class="example">
-SELECT * FROM iris_multi_model_info ORDER BY training_metrics_final DESC, training_loss_final;
-</pre>
-<pre class="result">
- mst_key | model_id | compile_params | fit_params | model_type | model_size | metrics_elapsed_time | metrics_type | training_metrics_final | training_loss_final | training_metrics | training_loss | validation_metrics_final | validation_loss_final | validation_metrics | validation_loss
----------+----------+---------------------------------------------------------------------------------+-----------------------+--------------+--------------+----------------------+--------------+------------------------+---------------------+---------------------+---------------------+--------------------------+-----------------------+--------------------+-----------------
- 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {119.42963886261} | {accuracy} | 0.983333349228 | 0.07286978513 | {0.983333349227905} | {0.072869785130024} | | | |
- 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {118.485460996628} | {accuracy} | 0.975000023842 | 0.0798489004374 | {0.975000023841858} | {0.079848900437355} | | | |
- 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {118.707404851913} | {accuracy} | 0.975000023842 | 0.143356323242 | {0.975000023841858} | {0.143356323242188} | | | |
- 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {118.224883794785} | {accuracy} | 0.958333313465 | 0.636615753174 | {0.958333313465118} | {0.636615753173828} | | | |
- 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {117.732690811157} | {accuracy} | 0.925000011921 | 0.161811202765 | {0.925000011920929} | {0.161811202764511} | | | |
- 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {120.357484817505} | {accuracy} | 0.833333313465 | 0.5542948246 | {0.833333313465118} | {0.55429482460022} | | | |
- 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {118.928852796555} | {accuracy} | 0.824999988079 | 0.301002770662 | {0.824999988079071} | {0.301002770662308} | | | |
- 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {120.566634893417} | {accuracy} | 0.816666662693 | 0.875298440456 | {0.816666662693024} | {0.87529844045639} | | | |
- 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {119.182703018188} | {accuracy} | 0.774999976158 | 0.785651266575 | {0.774999976158142} | {0.78565126657486} | | | |
- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {119.643137931824} | {accuracy} | 0.508333325386 | 0.762569189072 | {0.508333325386047} | {0.762569189071655} | | | |
- 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {120.15305685997} | {accuracy} | 0.333333343267 | 1.09794270992 | {0.333333343267441} | {1.09794270992279} | | | |
- 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {119.911739826202} | {accuracy} | 0.333333343267 | 1.10344016552 | {0.333333343267441} | {1.10344016551971} | | | |
-(12 rows)
-</pre>
-
--# Evaluate. Now run evaluate using models we built above:
-<pre class="example">
-DROP TABLE IF EXISTS iris_validate;
-SELECT madlib.madlib_keras_evaluate('iris_multi_model', -- model
- 'iris_test_packed', -- test table
- 'iris_validate', -- output table
- NULL, -- use gpus
- 3 -- mst_key to use
- );
-SELECT * FROM iris_validate;
-</pre>
-<pre class="result">
- loss | metric | metrics_type
--------------------+-------------------+--------------
- 0.103803977370262 | 0.966666638851166 | {accuracy}
-</pre>
-
--# Predict. Now predict using one of the models we built. We will use the validation data set
-for prediction as well, which is not usual but serves to show the syntax.
-The prediction is in the estimated_class_text column:
-<pre class="example">
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_multi_model', -- model
- 'iris_test', -- test_table
- 'id', -- id column
- 'attributes', -- independent var
- 'iris_predict', -- output table
- 'response', -- prediction type
- FALSE, -- use gpus
- 3 -- mst_key to use
- );
-SELECT * FROM iris_predict ORDER BY id;
-</pre>
-<pre class="result">
- id | estimated_class_text
------+----------------------
- 9 | Iris-setosa
- 18 | Iris-setosa
- 22 | Iris-setosa
- 26 | Iris-setosa
- 35 | Iris-setosa
- 38 | Iris-setosa
- 42 | Iris-setosa
- 43 | Iris-setosa
- 45 | Iris-setosa
- 46 | Iris-setosa
- 50 | Iris-setosa
- 53 | Iris-versicolor
- 60 | Iris-versicolor
- 68 | Iris-versicolor
- 77 | Iris-versicolor
- 78 | Iris-versicolor
- 79 | Iris-versicolor
- 81 | Iris-versicolor
- 82 | Iris-versicolor
- 85 | Iris-virginica
- 95 | Iris-versicolor
- 97 | Iris-versicolor
- 98 | Iris-versicolor
- 113 | Iris-virginica
- 117 | Iris-virginica
- 118 | Iris-virginica
- 127 | Iris-virginica
- 136 | Iris-virginica
- 143 | Iris-virginica
- 145 | Iris-virginica
-(30 rows)
-</pre>
-Count missclassifications:
-<pre class="example">
-SELECT COUNT(*) FROM iris_predict JOIN iris_test USING (id)
-WHERE iris_predict.estimated_class_text != iris_test.class_text;
-</pre>
-<pre class="result">
- count
--------+
- 1
-</pre>
-Percent missclassifications:
-<pre class="example">
-SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from
- (select iris_test.class_text as actual, iris_predict.estimated_class_text as estimated
- from iris_predict inner join iris_test
- on iris_test.id=iris_predict.id) q
-WHERE q.actual=q.estimated;
-</pre>
-<pre class="result">
- test_accuracy_percent
------------------------+
- 96.67
-</pre>
-
-<h4>Classification with Other Parameters</h4>
-
--# Validation dataset. Now use a validation dataset
-and compute metrics every 3rd iteration using
-the 'metrics_compute_frequency' parameter. This can
-help reduce run time if you do not need metrics
-computed at every iteration.
-<pre class="example">
-DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary, iris_multi_model_info;
-SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed', -- source_table
- 'iris_multi_model', -- model_output_table
- 'mst_table', -- model_selection_table
- 10, -- num_iterations
- FALSE, -- use gpus
- 'iris_test_packed', -- validation dataset
- 3, -- metrics compute frequency
- FALSE, -- warm start
- 'Sophie L.', -- name
- 'Model selection for iris dataset' -- description
- );
-</pre>
-View the model summary:
-<pre class="example">
-SELECT * FROM iris_multi_model_summary;
-</pre>
-<pre class="result">
-source_table | iris_train_packed
-validation_table | iris_test_packed
-model | iris_multi_model
-model_info | iris_multi_model_info
-dependent_varname | class_text
-independent_varname | attributes
-model_arch_table | model_arch_library
-num_iterations | 10
-metrics_compute_frequency | 3
-warm_start | f
-name | Sophie L.
-description | Model selection for iris dataset
-start_training_time | 2019-12-16 19:28:16.219137
-end_training_time | 2019-12-16 19:30:19.238692
-madlib_version | 1.17.0
-num_classes | 3
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype | character varying
-normalizing_const | 1
-metrics_iters | {3,6,9,10}
-</pre>
-View results for each model:
-<pre class="example">
-SELECT * FROM iris_multi_model_info ORDER BY training_metrics_final DESC, training_loss_final;
-</pre>
-<pre class="result">
- mst_key | model_id | compile_params | fit_params | model_type | model_size | metrics_elapsed_time | metrics_type | training_metrics_final | training_loss_final | training_metrics | training_loss | validation_metrics_final | validation_loss_final | validation_metrics | validation_loss
----------+----------+---------------------------------------------------------------------------------+-----------------------+--------------+--------------+-----------------------------------------------------------------------+--------------+------------------------+---------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+--------------------------+-----------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------
- 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {37.0420558452606,78.2046208381653,116.242669820786,134.287139892578} | {accuracy} | 0.975000023842 | 0.165132269263 | {0.75,0.958333313465118,0.958333313465118,0.975000023841858} | {0.618549585342407,0.319452553987503,0.223872095346451,0.165132269263268} | 0.966666638851 | 0.213689729571 | {0.733333349227905,0.933333337306976,0.933333337306976,0.966666638851166} | {0.683791160583496,0.370491921901703,0.255890935659409,0.213689729571342}
- 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {36.3931469917297,77.5780539512634,115.430645942688,133.599857807159} | {accuracy} | 0.966666638851 | 0.277698725462 | {0.591666638851166,0.966666638851166,0.666666686534882,0.966666638851166} | {0.634598553180695,0.334936827421188,0.615665555000305,0.27769872546196} | 0.966666638851 | 0.34405490756 | {0.5,0.966666638851166,0.566666662693024,0.966666638851166} | {0.643225967884064,0.41021603345871,0.805291295051575,0.344054907560349}
- 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {36.8482949733734,78.0155048370361,115.83317399025,134.079672813416} | {accuracy} | 0.958333313465 | 0.122385449708 | {0.883333325386047,0.941666662693024,0.858333349227905,0.958333313465118} | {0.291894346475601,0.146935686469078,0.270052850246429,0.122385449707508} | 0.933333337307 | 0.181496843696 | {0.766666650772095,0.866666674613953,0.899999976158142,0.933333337306976} | {0.395013928413391,0.245234906673431,0.301119148731232,0.181496843695641}
- 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {37.2318170070648,78.3925468921661,116.45490694046,134.491376876831} | {accuracy} | 0.941666662693 | 0.193545326591 | {0.966666638851166,0.941666662693024,0.941666662693024,0.941666662693024} | {0.39665362238884,0.213271111249924,0.190151125192642,0.193545326590538} | 0.933333337307 | 0.151459023356 | {1,0.966666638851166,0.933333337306976,0.933333337306976} | {0.464315593242645,0.198051139712334,0.138570576906204,0.151459023356438}
- 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {37.6678929328918,78.820240020752,116.939878940582,134.959810972214} | {accuracy} | 0.925000011921 | 0.192344605923 | {0.824999988079071,0.774999976158142,0.966666638851166,0.925000011920929} | {0.434513121843338,0.326292037963867,0.131333693861961,0.192344605922699} | 0.899999976158 | 0.209528595209 | {0.800000011920929,0.766666650772095,0.966666638851166,0.899999976158142} | {0.52033931016922,0.344535797834396,0.170280396938324,0.209528595209122}
- 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {38.0689258575439,79.4995639324188,117.36315202713,135.380483865738} | {accuracy} | 0.866666674614 | 0.390509605408 | {0.691666662693024,0.691666662693024,0.633333325386047,0.866666674613953} | {0.490214675664902,0.444783747196198,0.627961099147797,0.390509605407715} | 0.933333337307 | 0.376114845276 | {0.566666662693024,0.566666662693024,0.533333361148834,0.933333337306976} | {0.575542628765106,0.54660427570343,0.785183191299438,0.376114845275879}
- 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {38.474328994751,79.9709329605103,117.766183853149,135.803887844086} | {accuracy} | 0.841666638851 | 0.576696753502 | {0.616666674613953,0.699999988079071,0.758333325386047,0.841666638851166} | {0.90448260307312,0.750164151191711,0.616493880748749,0.576696753501892} | 0.899999976158 | 0.631914675236 | {0.666666686534882,0.699999988079071,0.733333349227905,0.899999976158142} | {0.871200919151306,0.780709445476532,0.665971457958221,0.631914675235748}
- 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {36.6214678287506,77.7987759113312,115.631717920303,133.83836388588} | {accuracy} | 0.758333325386 | 0.881635427475 | {0.308333337306976,0.316666662693024,0.75,0.758333325386047} | {1.12997460365295,1.02749967575073,0.923768699169159,0.881635427474976} | 0.766666650772 | 0.878168046474 | {0.433333337306976,0.433333337306976,0.766666650772095,0.766666650772095} | {1.07487094402313,0.974115014076233,0.916269063949585,0.878168046474457}
- 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {38.2849600315094,79.7524738311768,117.580325841904,135.606695890427} | {accuracy} | 0.691666662693 | 0.444524824619 | {0.908333361148834,0.391666680574417,0.691666662693024,0.691666662693024} | {0.335082054138184,2.02327847480774,0.444351017475128,0.444524824619293} | 0.566666662693 | 0.539750337601 | {0.800000011920929,0.266666680574417,0.566666662693024,0.566666662693024} | {0.433189332485199,2.3276960849762,0.534160375595093,0.539750337600708}
- 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {38.6593668460846,80.1789360046387,117.957875013351,135.995815992355} | {accuracy} | 0.683333337307 | 0.841839790344 | {0.316666662693024,0.366666674613953,0.666666686534882,0.683333337306976} | {1.07646071910858,0.963329672813416,0.87216705083847,0.841839790344238} | 0.666666686535 | 0.840192914009 | {0.433333337306976,0.533333361148834,0.666666686534882,0.666666686534882} | {1.02845978736877,0.941896677017212,0.861787617206573,0.840192914009094}
- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {37.8553328514099,79.2480089664459,117.139881849289,135.155915975571} | {accuracy} | 0.358333319426 | 1.11013436317 | {0.358333319425583,0.333333343267441,0.333333343267441,0.358333319425583} | {1.10554325580597,1.11694586277008,1.09756696224213,1.11013436317444} | 0.233333334327 | 1.17629003525 | {0.233333334326744,0.333333343267441,0.333333343267441,0.233333334326744} | {1.16081762313843,1.14324629306793,1.11625325679779,1.1762900352478}
- 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {37.4500079154968,78.6058378219604,116.700626850128,134.72905087471} | {accuracy} | 0.308333337307 | 1.06241953373 | {0.150000005960464,0.333333343267441,0.333333343267441,0.308333337306976} | {1.13338851928711,1.09694564342499,1.07030868530273,1.06241953372955} | 0.433333337307 | 1.03659796715 | {0.16666667163372,0.333333343267441,0.433333337306976,0.433333337306976} | {1.06262242794037,1.07252764701843,1.05843663215637,1.03659796714783}
-(12 rows)
-</pre>
-
--# Predict probabilities for each class:
-<pre class="example">
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_multi_model', -- model
- 'iris_test', -- test_table
- 'id', -- id column
- 'attributes', -- independent var
- 'iris_predict', -- output table
- 'prob', -- prediction type
- FALSE, -- use gpus
- 3 -- mst_key to use
- );
-SELECT * FROM iris_predict ORDER BY id;
-</pre>
-<pre class="result">
- id | prob_Iris-setosa | prob_Iris-versicolor | prob_Iris-virginica
------+------------------+----------------------+---------------------
- 9 | 0.99931216 | 0.00068789057 | 6.2587335e-10
- 18 | 0.99984336 | 0.00015656587 | 7.969957e-12
- 22 | 0.9998497 | 0.00015029701 | 6.4133347e-12
- 26 | 0.9995004 | 0.00049964694 | 2.2795305e-10
- 35 | 0.99964666 | 0.00035332117 | 9.4490485e-11
- 38 | 0.99964666 | 0.00035332117 | 9.4490485e-11
- 42 | 0.9985154 | 0.0014845316 | 5.293262e-09
- 43 | 0.99964476 | 0.0003552362 | 9.701174e-11
- 45 | 0.9997311 | 0.00026883607 | 3.076166e-11
- 46 | 0.9995486 | 0.00045140853 | 1.6814435e-10
- 50 | 0.9997856 | 0.00021441824 | 2.1316622e-11
- 53 | 9.837335e-06 | 0.97109175 | 0.028898403
- 60 | 0.00014028326 | 0.96552837 | 0.034331344
- 68 | 0.00087942625 | 0.9883348 | 0.010785843
- 77 | 6.08114e-06 | 0.94356424 | 0.056429718
- 78 | 7.116364e-07 | 0.8596206 | 0.14037873
- 79 | 1.3918722e-05 | 0.94052655 | 0.05945957
- 81 | 0.00045687397 | 0.9794796 | 0.020063542
- 82 | 0.0015463434 | 0.98768973 | 0.010763981
- 85 | 1.0929693e-05 | 0.87866926 | 0.121319845
- 95 | 6.3600986e-05 | 0.95264935 | 0.047287125
- 97 | 0.00020298029 | 0.981617 | 0.018180028
- 98 | 0.00019721613 | 0.98902065 | 0.01078211
- 113 | 1.0388683e-09 | 0.23626474 | 0.7637353
- 117 | 4.598902e-09 | 0.25669694 | 0.7433031
- 118 | 3.7139156e-11 | 0.13193987 | 0.8680601
- 127 | 2.1297862e-07 | 0.670349 | 0.32965073
- 136 | 7.1760774e-12 | 0.07074605 | 0.929254
- 143 | 1.2568385e-09 | 0.113820426 | 0.8861796
- 145 | 6.17019e-11 | 0.117578305 | 0.88242173
-(30 rows)
-</pre>
-
--# Warm start. Next, use the warm_start parameter
-to continue learning, using the coefficients from
-the run above. Note that we don't drop the
-model table or model summary table:
-<pre class="example">
-SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed', -- source_table
- 'iris_multi_model', -- model_output_table
- 'mst_table', -- model_selection_table
- 3, -- num_iterations
- FALSE, -- use gpus
- 'iris_test_packed', -- validation dataset
- 1, -- metrics compute frequency
- TRUE, -- warm start
- 'Sophie L.', -- name
- 'Simple MLP for iris dataset' -- description
- );
-SELECT * FROM iris_multi_model_summary;
-</pre>
-<pre class="result">
-source_table | iris_train_packed
-validation_table | iris_test_packed
-model | iris_multi_model
-model_info | iris_multi_model_info
-dependent_varname | class_text
-independent_varname | attributes
-model_arch_table | model_arch_library
-num_iterations | 3
-metrics_compute_frequency | 1
-warm_start | t
-name | Sophie L.
-description | Simple MLP for iris dataset
-start_training_time | 2019-12-16 20:07:41.488587
-end_training_time | 2019-12-16 20:08:27.20651
-madlib_version | 1.17.0
-num_classes | 3
-class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype | character varying
-normalizing_const | 1
-metrics_iters | {1,2,3}
-</pre>
-View results for each model:
-<pre class="example">
-SELECT * FROM iris_multi_model_info ORDER BY training_metrics_final DESC, training_loss_final;
-</pre>
-<pre class="result">
- mst_key | model_id | compile_params | fit_params | model_type | model_size | metrics_elapsed_time | metrics_type | training_metrics_final | training_loss_final | training_metrics | training_loss | validation_metrics_final | validation_loss_final | validation_metrics | validation_loss
----------+----------+---------------------------------------------------------------------------------+-----------------------+--------------+--------------+------------------------------------------------------+--------------+------------------------+---------------------+---------------------------------------------------------+----------------------------------------------------------+--------------------------+-----------------------+---------------------------------------------------------+---------------------------------------------------------
- 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {19.451014995575,37.2563629150391,54.7182998657227} | {accuracy} | 0.975000023842 | 0.490673750639 | {0.958333313465118,0.691666662693024,0.975000023841858} | {0.541427075862885,0.517450392246246,0.490673750638962} | 0.933333337307 | 0.557333409786 | {0.933333337306976,0.666666686534882,0.933333337306976} | {0.60710871219635,0.570206344127655,0.557333409786224}
- 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {18.2973220348358,36.3793680667877,54.0178129673004} | {accuracy} | 0.966666638851 | 0.0894369781017 | {0.966666638851166,0.966666638851166,0.966666638851166} | {0.133233144879341,0.111788973212242,0.0894369781017303} | 0.899999976158 | 0.195293620229 | {0.933333337306976,0.966666638851166,0.899999976158142} | {0.156044512987137,0.132803827524185,0.195293620228767}
- 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {17.6080539226532,35.6788699626923,53.3836889266968} | {accuracy} | 0.966666638851 | 0.147051945329 | {0.908333361148834,0.958333313465118,0.966666638851166} | {0.225205257534981,0.168186634778976,0.147051945328712} | 0.866666674614 | 0.250319689512 | {0.899999976158142,0.933333337306976,0.866666674613953} | {0.23467344045639,0.182851999998093,0.250319689512253}
- 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {18.7529940605164,36.8255958557129,54.3704080581665} | {accuracy} | 0.966666638851 | 0.244641214609 | {0.691666662693024,0.891666650772095,0.966666638851166} | {0.939713299274445,0.462556451559067,0.244641214609146} | 0.966666638851 | 0.298279434443 | {0.566666662693024,0.966666638851166,0.966666638851166} | {1.30671143531799,0.412235885858536,0.29827943444252}
- 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {17.4004180431366,35.4556438922882,53.1877279281616} | {accuracy} | 0.958333313465 | 0.123381219804 | {0.949999988079071,0.766666650772095,0.958333313465118} | {0.0919980704784393,0.576169073581696,0.123381219804287} | 0.933333337307 | 0.203262642026 | {0.866666674613953,0.766666650772095,0.933333337306976} | {0.199721112847328,0.959742486476898,0.203262642025948}
- 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {17.81547498703,35.8978669643402,53.5737180709839} | {accuracy} | 0.933333337307 | 0.150664463639 | {0.941666662693024,0.925000011920929,0.933333337306976} | {0.117781177163124,0.163000836968422,0.150664463639259} | 0.833333313465 | 0.365329563618 | {0.866666674613953,0.833333313465118,0.833333313465118} | {0.249404579401016,0.375173389911652,0.365329563617706}
- 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {19.686233997345,37.4543249607086,54.8708770275116} | {accuracy} | 0.858333349228 | 0.743227303028 | {0.675000011920929,0.708333313465118,0.858333349227905} | {0.808507084846497,0.774080872535706,0.743227303028107} | 0.966666638851 | 0.770158529282 | {0.666666686534882,0.666666686534882,0.966666638851166} | {0.808504283428192,0.791898012161255,0.770158529281616}
- 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {17.1583528518677,35.0312390327454,52.96133685112} | {accuracy} | 0.816666662693 | 0.739802956581 | {0.774999976158142,0.816666662693024,0.816666662693024} | {0.83727890253067,0.792884111404419,0.739802956581116} | 0.800000011921 | 0.758302807808 | {0.766666650772095,0.800000011920929,0.800000011920929} | {0.837629973888397,0.801746726036072,0.758302807807922}
- 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {16.9146749973297,34.794900894165,52.7328250408173} | {accuracy} | 0.808333337307 | 0.303489625454 | {0.683333337306976,0.966666638851166,0.808333337306976} | {1.05107569694519,0.189959138631821,0.303489625453949} | 0.866666674614 | 0.285375326872 | {0.666666686534882,0.966666638851166,0.866666674613953} | {1.01942157745361,0.238933652639389,0.285375326871872}
- 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {18.0590150356293,36.1394078731537,53.7930529117584} | {accuracy} | 0.699999988079 | 1.02253305912 | {0.550000011920929,0.691666662693024,0.699999988079071} | {1.0493084192276,1.03803598880768,1.02253305912018} | 0.666666686535 | 1.02013540268 | {0.633333325386047,0.600000023841858,0.666666686534882} | {1.03952574729919,1.03439521789551,1.02013540267944}
- 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {19.2141709327698,37.0566499233246,54.5629329681396} | {accuracy} | 0.691666662693 | 0.448221176863 | {0.691666662693024,0.691666662693024,0.691666662693024} | {0.447027385234833,0.444605946540833,0.448221176862717} | 0.566666662693 | 0.555035352707 | {0.566666662693024,0.566666662693024,0.566666662693024} | {0.551217257976532,0.540408432483673,0.555035352706909}
- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {18.501914024353,36.5938439369202,54.194118976593} | {accuracy} | 0.358333319426 | 1.09730923176 | {0.333333343267441,0.333333343267441,0.358333319425583} | {1.09999334812164,1.10405397415161,1.09730923175812} | 0.233333334327 | 1.12532019615 | {0.333333343267441,0.333333343267441,0.233333334326744} | {1.12446486949921,1.13782525062561,1.12532019615173}
-(12 rows)
-</pre>
-Note that the loss and accuracy values pick up from where the previous run left off.
-
-@anchor notes
-@par Notes
-
-1. Refer to the deep learning section of the Apache MADlib
-wiki [6] for important information including supported libraries
-and versions.
-
-2. Classification is currently supported, not regression.
-
-3. Reminder about the distinction between warm start and transfer learning. Warm start uses model
-state (weights) from the model output table from a previous training run -
-set the 'warm_start' parameter to TRUE in the fit function.
-Transfer learning uses initial model state (weights) stored in the 'model_arch_table' - in this case set the
-'warm_start' parameter to FALSE in the fit function.
-
-4. Here are some more details on how warm start works. These details are mostly applicable when implementing autoML algorithms on top of MADlib's model selection. In short, the 'model_selection_table' dictates which models get trained and output to the 'model_output_table' and associated summary and info tables. When 'warm_start' is TRUE, models are built for each 'mst_key' in the 'model_selection_table'. If there are prior runs for an 'mst_key' then the weights from that run will be used. If there are no prior runs for an 'mst_key' then random initialization will be used. For example, let's say we start with 'mst_keys' of 1, 2, 3, and 4 in the 'model_selection_table'. We run fit once to get model and info tables for 1, 2, 3, and 4. Then we modify the 'model_selection_table' as part of an autoML scheme, in which we remove the 'mst_key' for 1 and add a new 'mst_key' for 5. Next we run fit with warm start. The result will be models created for 'mst_keys' of 2, 3, 4, and 5. Warm start will be used for 2, 3, and 4 (using prior run) and random initialization will be used for 5 (no prior run). The 'mst_key' of 1 will be dropped.
-
-5. The 'num_iterations' parameter and the Keras fit parameter 'epochs' can substantially affect accuracy and run-time.
-In general, increasing the number of 'epochs' for a fixed 'num_iterations' will speed up training, but may result
-in lower accuracy. It's best to do some experimentation to find out what works for your models and dataset.
-
-@anchor background
-@par Technical Background
-
-For an introduction to deep learning foundations, including MLP and CNN,
-refer to [7].
-
-This module trains many models a time across the database cluster in order
-to explore network architectures and hyperparameters. It uses model hopper
-parallelism (MOP) and has high convergence efficiency since it does not do
-model averaging [2].
-
-On the effect of database cluster size: as the database cluster size increases,
-it will be faster to train a set of models, as long as you have at
-least as many model selection tuples as segments. This is because model state is "hopped" from
-segment to segment and training takes place in parallel [1,2]. If you have fewer model
-selection tuples to train than segments, then some
-segments may not be busy 100% of the time so speedup will not necessarily increase
-on a larger cluster. Inference (predict) is an embarrassingly parallel operation so
-inference runtimes will be proportionally faster as the number of segments increases.
-
-@anchor literature
-@literature
-
-@anchor mlp-lit-1
-[1] "Cerebro: Efficient and Reproducible Model Selection on Deep Learning Systems,"
-Supun Nakandala, Yuhao Zhang, and Arun Kumar, ACM SIGMOD 2019 DEEM Workshop,
-https://adalabucsd.github.io/papers/2019_Cerebro_DEEM.pdf
-
-[2] "Resource-Efficient and Reproducible Model Selection on Deep Learning Systems,"
-Supun Nakandala, Yuhao Zhang, and Arun Kumar, Technical Report, Computer Science and
-Engineering, University of California, San Diego
-https://adalabucsd.github.io/papers/TR_2019_Cerebro.pdf
-
-[3] https://keras.io/
-
-[4] https://www.tensorflow.org/
-
-[5] "Neural Networks for Machine Learning", Lectures 6a and 6b on mini-batch gradient descent,
-Geoffrey Hinton with Nitish Srivastava and Kevin Swersky,
-http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf
-
-[6] Deep learning section of Apache MADlib wiki https://cwiki.apache.org/confluence/display/MADLIB/Deep+Learning
-
-[7] Deep Learning, Ian Goodfellow, Yoshua Bengio and Aaron Courville, MIT Press, 2016.
-
-[8] Greenplum Database server configuration parameters https://gpdb.docs.pivotal.io/latest/ref_guide/config_params/guc-list.html
-
-@anchor related
-@par Related Topics
-
-File madlib_keras_fit_multiple_model.sql_in documents training, evaluate and predict functions.
-
-*/
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model(
- source_table VARCHAR,
- model_output_table VARCHAR,
- model_selection_table VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN,
- name VARCHAR,
- description VARCHAR
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras_fit_multiple_model')
- from utilities.control import SetGUC
- with AOControl(False):
- with SetGUC("plan_cache_mode", "force_generic_plan"):
- fit_obj = madlib_keras_fit_multiple_model.FitMultipleModel(**globals())
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model(
- source_table VARCHAR,
- model_output_table VARCHAR,
- model_selection_table VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN,
- name VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model(
- source_table VARCHAR,
- model_output_table VARCHAR,
- model_selection_table VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER,
- warm_start BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model(
- source_table VARCHAR,
- model_output_table VARCHAR,
- model_selection_table VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR,
- metrics_compute_frequency INTEGER
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, $7, FALSE, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model(
- source_table VARCHAR,
- model_output_table VARCHAR,
- model_selection_table VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN,
- validation_table VARCHAR
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, NULL, FALSE, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model(
- source_table VARCHAR,
- model_output_table VARCHAR,
- model_selection_table VARCHAR,
- num_iterations INTEGER,
- use_gpus BOOLEAN
-) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, NULL, NULL, FALSE, NULL, NULL);
-$$ LANGUAGE sql VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_transition_multiple_model(
- state BYTEA,
- dependent_var BYTEA,
- independent_var BYTEA,
- dependent_var_shape INTEGER[],
- independent_var_shape INTEGER[],
- model_architecture TEXT,
- compile_params TEXT,
- fit_params TEXT,
- dist_key INTEGER,
- dist_key_mapping INTEGER[],
- current_seg_id INTEGER,
- segments_per_host INTEGER,
- images_per_seg INTEGER[],
- use_gpus BOOLEAN,
- accessible_gpus_for_seg INTEGER[],
- prev_serialized_weights BYTEA,
- is_final_iteration BOOLEAN
-) RETURNS BYTEA AS $$
-PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras')
- return madlib_keras.fit_transition(is_multiple_model = True, **globals())
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fit_step_multiple_model(
- BYTEA,
- BYTEA,
- INTEGER[],
- INTEGER[],
- TEXT,
- TEXT,
- TEXT,
- INTEGER,
- INTEGER[],
- INTEGER,
- INTEGER,
- INTEGER[],
- BOOLEAN,
- INTEGER[],
- BYTEA,
- BOOLEAN);
-CREATE AGGREGATE MADLIB_SCHEMA.fit_step_multiple_model(
- /* dependent_var */ BYTEA,
- /* independent_var */ BYTEA,
- /* dependent_var_shape */ INTEGER[],
- /* independent_var_shape */ INTEGER[],
- /* model_architecture */ TEXT,
- /* compile_params */ TEXT,
- /* fit_params */ TEXT,
- /* dist_key */ INTEGER,
- /* dist_key_mapping */ INTEGER[],
- /* current_seg_id */ INTEGER,
- /* segments_per_host */ INTEGER,
- /* images_per_seg */ INTEGER[],
- /* use_gpus */ BOOLEAN,
- /* accessible_gpus_for_seg */ INTEGER[],
- /* prev_serialized_weights */ BYTEA,
- /* is_final_iteration */ BOOLEAN
-)(
- STYPE=BYTEA,
- SFUNC=MADLIB_SCHEMA.fit_transition_multiple_model
-);
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.py_in
deleted file mode 100644
index 4eec8a7..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.py_in
+++ /dev/null
@@ -1,256 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-@file madlib_keras_gpu_info.py_in
-
-@brief GPU configuration helper function
-
-@namespace madlib_keras_gpu_info
-"""
-
-import os
-import subprocess
-
-import plpy
-from utilities.utilities import is_platform_pg
-from utilities.utilities import unique_string
-from utilities.validate_args import output_tbl_valid
-
-class OutputInfoSchema:
- TEMP_INFO_TABLE = unique_string(desp='gpu_info')
- SEG_ID_COL = 'gp_seg_id'
- GPU_DESCR_COL = 'gpu_descr'
-
-
-class Source:
- NVIDIA = 'nvidia'
- TENSORFLOW = 'tensorflow'
-
-
-class GPUInfoFunctions:
- @staticmethod
- def get_gpu_info_from_nvidia():
- """
- This function will run only on segment(s). Make sure not to run any non
- select plpy execute.
- :return: list of gpu descriptions as returned by nvidia-smi -L.
- """
- try:
- return subprocess.check_output(["nvidia-smi", "-L"]).splitlines()
- except Exception: # Handle case when nvidia-smi -L fails
- return []
-
- @staticmethod
- def get_gpu_info_from_tensorflow():
- """
- This function will run only on segment(s). Make sure not to run any non
- select plpy execute.
- :return: list of gpu descriptions as returned by tensorflow
- """
- current_working_dir = os.path.dirname(os.path.realpath(__file__))
- gpus = subprocess.check_output(["python", "gpu_info_from_tf.py"],
- cwd=current_working_dir).splitlines()
- return gpus
-
-
-def gpu_configuration(schema_madlib, output_table, source):
- """
- :return: List of gpus along with their hostname in the format
- GPDB
- gpu_descr | hostname
- ------------------+--------------------------
- NVIDIA Tesla P100 | host1
- NVIDIA Tesla P100 | host1
- Super Duper GPU | host2
- Super Duper GPU | host2
- 1. We use gp_dist_random to run either the nvidia smi UDF or the tensorflow UDF
- on all the hosts. (see gpu_info_nvidia/gpu_info_tensorflow)
- 2. Also we do not need to run the tf/nvidia UDF on all the segments,
- just one segment per host. That's why we group the output of
- gp_segment_configuration by hostname and get the min segment from
- each host.
- 3. To get the hostname along with the gpu description, we have to join
- the output of nvidia/tf UDF with gp_segment_configuration and filter
- out the following
- * master
- * mirror segments
- * empty/null gpu description
- Workflow for gpdb
- 1. Run query to get min seg ids on each host. This is so that we can run
- the gpu UDF on just one segment per host.
- 2. Create a table by running the tf/nvidia UDF on the segment ids returned
- from the previous step. Note that this table will only contain the output
- of the UDF and the segment id itself. This table does not contain hostnames
- 3. To get the hostname associated with the segment id, we need to join the
- table created in step with gp_segment_configuration.
- It's important to note that we can merge all these 3 queries into one but
- the problem with that is that a redistribution happens before running the UDF
- which means the UDF does not run on the segments that we pass in to the query.
- To avoid this, we broke down the query into 3 parts so that the UDF is always
- run on the intended segments.
-
- POSTGRES
- gpu_descr | hostname
- ------------------+--------------------------
- NVIDIA Tesla P100 | localhost
- """
- module_name = 'madlib_keras_gpu_info'
- output_tbl_valid(output_table, module_name)
-
- if not source:
- source = Source.TENSORFLOW
- source = source.lower()
- if source != Source.TENSORFLOW and source != Source.NVIDIA:
- plpy.error("DL: source has to be one of {0} or {1}".format(
- Source.TENSORFLOW, Source.NVIDIA))
-
- gpu_fn_name = 'gpu_info_{0}'.format(source)
- if is_platform_pg():
- gpu_for_postgres(schema_madlib, output_table, gpu_fn_name)
- else:
- gpu_for_gpdb(schema_madlib, output_table, gpu_fn_name)
-
-
-def gpu_for_postgres(schema_madlib, output_table, gpu_fn_name):
- gpu_info_query = """
- CREATE TABLE {0} AS
- SELECT 'localhost' as hostname, {1} from (SELECT unnest({2}.{3}()) AS {1}) s1
- where {1} is NOT NULL AND {1} != ''
- """.format(output_table, OutputInfoSchema.GPU_DESCR_COL,
- schema_madlib, gpu_fn_name)
- plpy.execute(gpu_info_query)
-
-
-def gpu_for_gpdb(schema_madlib, output_table, gpu_fn_name):
- min_seg_on_each_host = get_min_seg_ids_on_each_host()
-
- create_gpu_info_table_without_hostname(schema_madlib, gpu_fn_name,
- min_seg_on_each_host)
-
- create_gpu_info_table_with_hostname(output_table)
-
- plpy.execute("DROP TABLE IF EXISTS {0}".format(OutputInfoSchema.TEMP_INFO_TABLE))
-
-
-def get_min_seg_ids_on_each_host():
- """
- Run query to get min seg ids on each host. This is so that we can run
- the gpu UDF on just one segment per host.
- :return: List of min seg id per host
- """
- min_seg_id_alias = 'min_seg_id'
- min_seg_query = """
- SELECT {min_seg_id_alias} FROM
- (select hostname, min(content) AS {min_seg_id_alias}
- FROM gp_segment_configuration WHERE content != -1 AND role='p'
- GROUP BY hostname) min_seg_id_subquery
- """.format(**locals())
- min_seg_on_each_host = plpy.execute(min_seg_query)
-
- min_seg_on_each_host = ','.join([str(seg[min_seg_id_alias])
- for seg in min_seg_on_each_host])
- return min_seg_on_each_host
-
-
-def create_gpu_info_table_without_hostname(schema_madlib, gpu_fn_name,
- min_seg_on_each_host):
- """ output_table,
- Create a table by running the tf/nvidia UDF on the segment ids returned
- from the previous step. Note that this table will only contain the output
- of the UDF and the segment id itself. This table does not contain hostnames
- :param schema_madlib:
- :param gpu_fn_name:
- :param min_seg_on_each_host:
- """
- gpu_info_per_host_query = """
- CREATE TABLE {0} AS SELECT gp_segment_id AS {1}, {2}.{3}()
- AS {4} FROM gp_dist_random('gp_id') WHERE gp_segment_id IN ({5})
- """.format(OutputInfoSchema.TEMP_INFO_TABLE,
- OutputInfoSchema.SEG_ID_COL,
- schema_madlib, gpu_fn_name,
- OutputInfoSchema.GPU_DESCR_COL,
- min_seg_on_each_host)
- plpy.execute(gpu_info_per_host_query)
-
-
-def create_gpu_info_table_with_hostname(output_table):
- """
- Create the final output table that contains the hostname and the gpu description.
- To create this table, we need to join the table created in
- create_gpu_info_table_without_hostname with gp_segment_configuration.
- """
- final_join_query = """
- CREATE TABLE {0} AS
- SELECT hostname, {1} FROM
- (
- SELECT hostname, unnest({1}) AS {1} FROM {2}
- JOIN
- gp_segment_configuration ON {3}=content WHERE content != -1 AND role='p'
- ) s1
- WHERE {1} != '' AND {1} is NOT NULL ORDER BY 1,2;
- """.format(output_table, OutputInfoSchema.GPU_DESCR_COL,
- OutputInfoSchema.TEMP_INFO_TABLE,
- OutputInfoSchema.SEG_ID_COL)
- plpy.execute(final_join_query)
-
-
-def gpu_configuration_help(schema_madlib):
- """
- Help function for gpu configuration
-
- Args:
- @param schema_madlib
-
- Returns:
- String. Help/usage information
- """
-
- help_string = """
- Utility function to report number and type of GPUs on the database cluster.
- -----------------------------------------------------------------------
- USAGE
- -----------------------------------------------------------------------
- SELECT {schema_madlib}.gpu_configuration(
- output_table, -- Name of the output table to write out the
- GPU information.
- source -- Default: 'tensorflow'. Source for determining
- GPU configuration.
- Using 'tensorflow' returns a description based
- on what TensorFlow reports.
- Using 'nvidia' returns a description based
- on what the Nvidia Systems Management Interface
- (nvidia-smi) reports [1].
- Note that MADlib and Keras will use the TensorFlow
- information; the lower level nvidia-smi info
- is provided for convenience.
- )
- );
-
- -----------------------------------------------------------------------
- OUTPUT
- -----------------------------------------------------------------------
- The output table ('output_table' above) contains the following columns:
-
- hostname: Name of the host machine in the cluster.
- Does not include master or mirrors. For PostgreSQL this will
- always return 'localhost'.
- gpu_descr: String reported by TensorFlow or nvidia-smi.
- """
-
- return help_string.format(schema_madlib=schema_madlib)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.sql_in
deleted file mode 100644
index d2418e4..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.sql_in
+++ /dev/null
@@ -1,279 +0,0 @@
-/* ----------------------------------------------------------------------- *//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *
- * @file madlib_keras_gpu_info.sql_in
- *
- * @brief SQL functions for GPU configuration.
- * @date Nov 2019
- *
- *
- *//* ----------------------------------------------------------------------- */
-
-m4_include(`SQLCommon.m4')
-
-/**
-@addtogroup grp_gpu_configuration
-
-@brief Utility function to report number and type of GPUs in the database cluster.
-
-\warning <em> This MADlib method is still in early stage development.
-Interface and implementation are subject to change. </em>
-
-<div class="toc"><b>Contents</b><ul>
-<li class="level1"><a href="#get_gpu_config">GPU Configuration</a></li>
-<li class="level1"><a href="#example">Examples</a></li>
-<li class="level1"><a href="#references">References</a></li>
-<li class="level1"><a href="#related">Related Topics</a></li>
-</ul></div>
-
-This utility function reports the number and type of GPUs
-attached to hosts on the database cluster.
-This can be useful when determining which segments to use for
-training deep neural nets. For example, for economic reasons
-you may wish to set up a heterogeneous clusters with GPUs only
-on some of the hosts, not all of them. This utility
-can help you identify where the GPUS are and direct the compute
-to those locations only in subsequent training and evaluation steps.
-
-@anchor get_gpu_config
-@par GPU Confuguration
-
-<pre class="syntax">
-gpu_configuration( output_table,
- source
- )
-</pre>
-\b Arguments
-<dl class="arglist">
- <dt>output_table</dt>
- <dd>TEXT. Name of the output table to write out the GPU information.
- </dd>
-
- <dt>source (optional)</dt>
- <dd>TEXT, default: 'tensorflow'. Source for determining GPU configuration.
- Using 'tensorflow' returns a description based on what TensorFlow reports.
- Using 'nvidia' returns a description based on what the Nvidia Systems
- Management Interface (nvidia-smi) reports [1]. Note that MADlib and Keras will use the
- TensorFlow information; the lower level nvidia-smi info is provided for convenience.
-
- @note
- If the command 'nvidia-smi -L' returns an error, we do not pass through
- the error message, but instead will show no GPUs for that host in the output table.
- You may want to run nvidia-smi from the command line to see error
- and informational messages.
-
- </dd>
-</dl>
-
-<b>Output</b>
-<br>
- The output table contains the following:
- <table class="output">
- <tr>
- <th>hostname</th>
- <td>TEXT. Name of the host machine in the cluster.
- Does not include master or mirrors. For PostgreSQL
- this will always return 'localhost'.
- </td>
- </tr>
- <tr>
- <th>gpu_descr</th>
- <td>TEXT. String reported by TensorFlow or nvidia-smi.
- The formats are different and shown in the examples below.
- </td>
- </tr>
- </table>
-</br>
-
-@anchor example
-@par Examples
-
--# Get GPU configuration as per TensorFlow:
-<pre class="example">
-DROP TABLE IF EXISTS host_gpu_mapping_tf;
-SELECT * FROM madlib.gpu_configuration('host_gpu_mapping_tf');
-SELECT * FROM host_gpu_mapping_tf ORDER BY hostname, gpu_descr;
-</pre>
-<pre class="result">
- hostname | gpu_descr
-----------+------------------------------------------------------------------------------------------
- phoenix0 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix0 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix0 | device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0
- phoenix0 | device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0
- phoenix1 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix1 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix3 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix3 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix3 | device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0
- phoenix3 | device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0
- phoenix4 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix4 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix4 | device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0
- phoenix4 | device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0
-(14 rows)
-</pre>
-In this heterogeneous cluster there are 4 GPUs attached to hosts 0, 3 and 4. There are 2 GPUs
-attached to host 1 and no GPUs attached to host 2.
-
--# Get GPU configuration as per nvidia-smi:
-<pre class="example">
-DROP TABLE IF EXISTS host_gpu_mapping_nvidia;
-SELECT * FROM madlib.gpu_configuration('host_gpu_mapping_nvidia', -- output table
- 'nvidia' -- source for GPU info
- );
-SELECT * FROM host_gpu_mapping_nvidia ORDER BY hostname, gpu_descr;
-</pre>
-<pre class="result">
- hostname | gpu_descr
-----------+------------------------------------------------------------------------------
- phoenix0 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-f2ccc77e-2501-f6ee-4754-069dda256fb2)
- phoenix0 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-b1fc40ca-c7c6-bc86-f20f-6e9a62cda3f8)
- phoenix0 | GPU 2: Tesla P100-PCIE-16GB (UUID: GPU-d93bb21b-96f9-7c1d-3bab-cdd92b7bbc9d)
- phoenix0 | GPU 3: Tesla P100-PCIE-16GB (UUID: GPU-2d79c4a8-479e-2f33-39f8-3ba80b63f830)
- phoenix1 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-0af6bb1e-5b5b-4988-ad3a-a917e9584702)
- phoenix1 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-d824c976-a8aa-ef26-a13c-9a9a7fe86bfd)
- phoenix3 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-3681d0b6-1ec6-0453-fd81-29d88e549cd9)
- phoenix3 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-d4b1f2e7-b238-ac9a-bbfe-918adeb69472)
- phoenix3 | GPU 2: Tesla P100-PCIE-16GB (UUID: GPU-42a32ef1-a60c-e599-c8cf-0e669111ab6f)
- phoenix3 | GPU 3: Tesla P100-PCIE-16GB (UUID: GPU-1cce09c4-6856-8031-be0b-8e8bbf9a10f3)
- phoenix4 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-a71bdc18-fdd5-ba25-617e-19b23cc8e827)
- phoenix4 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-f9d13688-7fe6-a029-24d1-985a5659f18f)
- phoenix4 | GPU 2: Tesla P100-PCIE-16GB (UUID: GPU-06a7f54b-c07a-e87a-20d6-09bd99b19531)
- phoenix4 | GPU 3: Tesla P100-PCIE-16GB (UUID: GPU-af3b32f3-8bd9-cb75-a8fb-25253b9da926)
-(14 rows)
-</pre>
-
--# To get a fuller picture at the segment level, combine with the Greenplum catalog
-table 'gp_segment_configuration' which contains information about
-segment instance configuration [2]. Here is an example of this table
-filtering out master and mirrors:
-<pre class="example">
-SELECT * FROM gp_segment_configuration WHERE role='p' AND content>=0 ORDER BY hostname, dbid;
-</pre>
-<pre class="result">
- dbid | content | role | preferred_role | mode | status | port | hostname | address | replication_port
-------+---------+------+----------------+------+--------+-------+----------+----------+------------------
- 2 | 0 | p | p | c | u | 40000 | phoenix0 | phoenix0 | 70000
- 3 | 1 | p | p | c | u | 40001 | phoenix0 | phoenix0 | 70001
- 4 | 2 | p | p | c | u | 40002 | phoenix0 | phoenix0 | 70002
- 5 | 3 | p | p | c | u | 40003 | phoenix0 | phoenix0 | 70003
- 6 | 4 | p | p | c | u | 40000 | phoenix1 | phoenix1 | 70000
- 7 | 5 | p | p | c | u | 40001 | phoenix1 | phoenix1 | 70001
- 8 | 6 | p | p | c | u | 40002 | phoenix1 | phoenix1 | 70002
- 9 | 7 | p | p | c | u | 40003 | phoenix1 | phoenix1 | 70003
- 10 | 8 | p | p | c | u | 40000 | phoenix2 | phoenix2 | 70000
- 11 | 9 | p | p | c | u | 40001 | phoenix2 | phoenix2 | 70001
- 12 | 10 | p | p | c | u | 40002 | phoenix2 | phoenix2 | 70002
- 13 | 11 | p | p | c | u | 40003 | phoenix2 | phoenix2 | 70003
- 14 | 12 | p | p | c | u | 40000 | phoenix3 | phoenix3 | 70000
- 15 | 13 | p | p | c | u | 40001 | phoenix3 | phoenix3 | 70001
- 16 | 14 | p | p | c | u | 40002 | phoenix3 | phoenix3 | 70002
- 17 | 15 | p | p | c | u | 40003 | phoenix3 | phoenix3 | 70003
- 18 | 16 | p | p | c | u | 40000 | phoenix4 | phoenix4 | 70000
- 19 | 17 | p | p | c | u | 40001 | phoenix4 | phoenix4 | 70001
- 20 | 18 | p | p | c | u | 40002 | phoenix4 | phoenix4 | 70002
- 21 | 19 | p | p | c | u | 40003 | phoenix4 | phoenix4 | 70003
-(20 rows)
-</pre>
-Now join this table with the GPU resources table to create a table containing a
-list of all segments on hosts with GPUs attached:
-<pre class="example">
-DROP TABLE IF EXISTS segments_to_use;
-CREATE TABLE segments_to_use AS
- SELECT DISTINCT dbid, hostname FROM gp_segment_configuration JOIN host_gpu_mapping_tf USING (hostname)
- WHERE role='p' AND content>=0;
-SELECT * FROM segments_to_use ORDER BY hostname, dbid;
-</pre>
-<pre class="result">
- dbid | hostname
-------+----------
- 2 | phoenix0
- 3 | phoenix0
- 4 | phoenix0
- 5 | phoenix0
- 6 | phoenix1
- 7 | phoenix1
- 8 | phoenix1
- 9 | phoenix1
- 14 | phoenix3
- 15 | phoenix3
- 16 | phoenix3
- 17 | phoenix3
- 18 | phoenix4
- 19 | phoenix4
- 20 | phoenix4
- 21 | phoenix4
-(16 rows)
-</pre>
-
-@anchor references
-@par References
-
-[1] Nvidia Systems Management Interface (nvidia-smi) https://developer.nvidia.com/nvidia-system-management-interface
-
-[2] Greenplum 'gp_segment_configuration' table https://gpdb.docs.pivotal.io/latest/ref_guide/system_catalogs/gp_segment_configuration.html
-
-@anchor related
-@par Related Topics
-
-See madlib_keras_gpu_info.sql_in
-
-*/
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_info_tensorflow() RETURNS TEXT[] as
-$$
- PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras_gpu_info')
- return madlib_keras_gpu_info.GPUInfoFunctions.get_gpu_info_from_tensorflow()
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_info_nvidia() RETURNS TEXT[] as
-$$
- PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras_gpu_info')
- return madlib_keras_gpu_info.GPUInfoFunctions.get_gpu_info_from_nvidia()
-$$ LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_configuration(output_table text, source text)
-RETURNS VOID AS
-$$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras_gpu_info')
- from utilities.control import MinWarning
- with AOControl(False) and MinWarning("error"):
- madlib_keras_gpu_info.gpu_configuration(schema_madlib, output_table, source)
-$$
-LANGUAGE plpythonu
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_configuration(output_table text)
-RETURNS VOID AS
-$$
- SELECT MADLIB_SCHEMA.gpu_configuration($1, NULL);
-$$
-LANGUAGE sql;
-
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_configuration()
- RETURNS VARCHAR AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras_gpu_info')
- return madlib_keras_gpu_info.gpu_configuration_help(schema_madlib)
-$$ LANGUAGE plpythonu IMMUTABLE
- m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
deleted file mode 100644
index 5be078b..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
+++ /dev/null
@@ -1,316 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import numpy as np
-from model_arch_info import ModelArchSchema
-from utilities.utilities import add_postfix
-from utilities.utilities import unique_string
-from utilities.utilities import is_platform_pg
-from utilities.validate_args import table_exists
-from madlib_keras_gpu_info import GPUInfoFunctions
-import plpy
-
-
-############### Constants used in other deep learning files #########
-# Name of columns in model summary table.
-CLASS_VALUES_COLNAME = "class_values"
-NORMALIZING_CONST_COLNAME = "normalizing_const"
-COMPILE_PARAMS_COLNAME = "compile_params"
-DEPENDENT_VARNAME_COLNAME = "dependent_varname"
-DEPENDENT_VARTYPE_COLNAME = "dependent_vartype"
-INDEPENDENT_VARNAME_COLNAME = "independent_varname"
-MODEL_ARCH_TABLE_COLNAME = "model_arch_table"
-MODEL_ID_COLNAME = ModelArchSchema.MODEL_ID
-MODEL_WEIGHTS_COLNAME = "model_weights"
-METRIC_TYPE_COLNAME = "metrics_type"
-
-# Name of independent, dependent and distribution key colnames in batched table.
-# These are readonly variables, do not modify.
-# MADLIB-1300 Adding these variables for DL only at this time.
-MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL = "dependent_var"
-MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL = "independent_var"
-DISTRIBUTION_KEY_COLNAME = "__dist_key__"
-## sql variable types
-FLOAT32_SQL_TYPE = 'REAL'
-SMALLINT_SQL_TYPE = 'SMALLINT'
-
-DEFAULT_NORMALIZING_CONST = 1.0
-GP_SEGMENT_ID_COLNAME = "gp_segment_id"
-INTERNAL_GPU_CONFIG = '__internal_gpu_config__'
-
-#####################################################################
-
-# Prepend a dimension to np arrays using expand_dims.
-def expand_input_dims(input_data):
- input_data = np.array(input_data, dtype=np.float32)
- input_data = np.expand_dims(input_data, axis=0)
- return input_data
-
-def np_array_float32(var, var_shape):
- arr = np.frombuffer(var, dtype=np.float32)
- arr.shape = var_shape
- return arr
-
-def np_array_int16(var, var_shape):
- arr = np.frombuffer(var, dtype=np.int16)
- arr.shape = var_shape
- return arr
-
-def strip_trailing_nulls_from_class_values(class_values):
- """
- class_values is a list of unique class levels in training data. This
- could have multiple Nones in it, and this function strips out all the
- Nones that occur after the first element in the list.
- Examples:
- 1) input class_values = ['cat', 'dog']
- output class_values = ['cat', 'dog']
-
- 2) input class_values = [None, 'cat', 'dog']
- output class_values = [None, 'cat', 'dog']
-
- 3) input class_values = [None, 'cat', 'dog', None, None]
- output class_values = [None, 'cat', 'dog']
-
- 4) input class_values = ['cat', 'dog', None, None]
- output class_values = ['cat', 'dog']
-
- 5) input class_values = [None, None]
- output class_values = [None]
- @args:
- @param: class_values, list
- @returns:
- updated class_values list
- """
- num_of_valid_class_values = 0
- if class_values is not None:
- for ele in class_values:
- if ele is None and num_of_valid_class_values > 0:
- break
- num_of_valid_class_values += 1
- # Pass only the valid class_values for creating columns
- class_values = class_values[:num_of_valid_class_values]
- return class_values
-
-def get_image_count_per_seg_from_array(current_seg_id, images_per_seg):
- """
- Get the image count from the array containing all the images
- per segment. Based on the platform, we find the index of the current segment.
- This function is only called from inside the transition function.
- """
- if is_platform_pg():
- total_images = images_per_seg[0]
- else:
- total_images = images_per_seg[current_seg_id]
- return total_images
-
-def get_image_count_per_seg_for_minibatched_data_from_db(table_name):
- """
- Query the given minibatch formatted table and return the total rows per segment.
- Since we cannot pass a dictionary to the keras fit step function we create
- arrays out of the segment numbers and the rows per segment values.
- This function assumes that the table is not empty and is minibatched which means
- that it would have been distributed by __dist_key__.
- :param table_name:
- :return: Returns two arrays
- 1. An array containing all the segment numbers in ascending order
- 1. An array containing the total images on each of the segments in the
- segment array.
- """
-
- mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
-
- shape_col = add_postfix(mb_dep_var_col, "_shape")
-
- if is_platform_pg():
- res = plpy.execute(
- """ SELECT {0}::SMALLINT[] AS shape
- FROM {1}
- """.format(shape_col, table_name))
- images_per_seg = [sum(r['shape'][0] for r in res)]
- seg_ids = [0]
- else:
- # The number of images in the buffer is the first dimension in the shape.
- # Using __dist_key__ instead of gp_segment_id: Since gp_segment_id is
- # not the actual distribution key column, the optimizer/planner
- # generates a plan with Redistribute Motion, creating multiple slices on
- # each segment. For DL, since GPU memory allocation is tied to the process
- # where it is initialized, we want to minimize creating any additional
- # slices per segment. This is mainly to avoid any GPU memory allocation
- # failures which can occur when a newly created slice(process) tries
- # allocating GPU memory which is already allocated by a previously
- # created slice(process).
- # Since the minibatch_preprocessor evenly distributes the data with __dist_key__
- # as the input table's distribution key, using this for calculating
- # total images on each segment will avoid creating unnecessary slices(processes).
- images_per_seg = plpy.execute(
- """ SELECT {0}, sum({1}[1]) AS images_per_seg
- FROM {2}
- GROUP BY {0}
- """.format(DISTRIBUTION_KEY_COLNAME, shape_col, table_name))
- seg_ids = [int(each_segment[DISTRIBUTION_KEY_COLNAME])
- for each_segment in images_per_seg]
- images_per_seg = [int(each_segment["images_per_seg"])
- for each_segment in images_per_seg]
-
- return seg_ids, images_per_seg
-
-def get_image_count_per_seg_for_non_minibatched_data_from_db(table_name):
- """
- Query the given non minibatch formatted table and return the total rows per segment.
- Since we cannot pass a dictionary to the keras fit step function we create arrays
- out of the segment numbers and the rows per segment values.
- This function assumes that the table is not empty.
- :param table_name:
- :return: gp segment id col name and two arrays
- 1. An array containing all the segment numbers in ascending order
- 2. An array containing the total rows for each of the segments in the
- segment array
- """
- if is_platform_pg():
- images_per_seg = plpy.execute(
- """ SELECT count(*) AS images_per_seg
- FROM {0}
- """.format(table_name))
- seg_ids = [0]
- gp_segment_id_col = '0'
- else:
- # Compute total buffers on each segment
- images_per_seg = plpy.execute(
- """ SELECT {0}, count(*) AS images_per_seg
- FROM {1}
- GROUP BY {0}
- """.format(GP_SEGMENT_ID_COLNAME, table_name))
- seg_ids = [int(image[GP_SEGMENT_ID_COLNAME]) for image in images_per_seg]
- gp_segment_id_col = '{0}.{1}'.format(table_name,GP_SEGMENT_ID_COLNAME)
-
- images_per_seg = [int(image["images_per_seg"]) for image in images_per_seg]
- return gp_segment_id_col, seg_ids, images_per_seg
-
-def parse_shape(shape):
- # Parse the shape format given by the sql into an int array
- # [1:10][1:32][1:3] -> [10, 32, 3]
- # Split on :, discard the first one [1:],
- # split each piece on ], take the first piece [0], convert to int
- return [int(a.split(']')[0]) for a in shape.split(':')[1:]]
-
-
-def query_model_configs(model_selection_table, model_selection_summary_table,
- mst_key_col, model_arch_table_col):
- msts_query = """
- SELECT * FROM {model_selection_table}
- ORDER BY {mst_key_col}
- """.format(**locals())
- model_arch_table_query = """
- SELECT {model_arch_table_col}
- FROM {model_selection_summary_table}
- """.format(**locals())
- msts = list(plpy.execute(msts_query))
- model_arch_table = plpy.execute(model_arch_table_query)[0][model_arch_table_col]
- return msts, model_arch_table
-
-def query_dist_keys(source_table, dist_key_col):
- """ Read distinct keys from the source table """
- dist_key_query = """
- SELECT DISTINCT({dist_key_col}) FROM {source_table}
- ORDER BY {dist_key_col}
- """.format(dist_key_col=dist_key_col,
- source_table=source_table)
- res = list(plpy.execute(dist_key_query))
- res = [x[dist_key_col] for x in res]
- return res
-
-def query_weights(model_output_table, model_weights_col, mst_key_col, mst_key):
- mlp_weights_query = """
- SELECT {model_weights_col}, {mst_key_col}
- FROM {model_output_table}
- WHERE {mst_key_col} = {mst_key}
- """.format(**locals())
-
- res = plpy.execute(mlp_weights_query)
- return res[0][model_weights_col]
-
-def create_summary_view(module_name, model_table, mst_key):
- tmp_view_summary = unique_string('tmp_view_summary')
- model_summary_table = add_postfix(model_table, "_summary")
- model_info_table = add_postfix(model_table, "_info")
- if not (table_exists(model_summary_table) and
- table_exists(model_info_table)):
- plpy.error("{0}: Missing summary and/or info tables for {1}".format(
- module_name, model_table))
-
- res = plpy.execute("""
- SELECT mst_key FROM {model_info_table} WHERE mst_key = {mst_key}
- """.format(**locals()))
- if len(res) < 1:
- plpy.error("{0}: mst_key {1} does not exist in the info table".format(
- module_name, mst_key))
-
- plpy.execute("""
- CREATE VIEW {tmp_view_summary} AS
- SELECT *
- FROM {model_summary_table}, {model_info_table}
- WHERE mst_key = {mst_key}
- """.format(**locals()))
- return tmp_view_summary
-
-
-def get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name):
- if is_platform_pg():
- gpus = GPUInfoFunctions.get_gpu_info_from_tensorflow()
- if not gpus:
- plpy.error("{0} error: No GPUs configured on host.".format(module_name))
- return [len(gpus)]
- else:
- gpu_info_table = unique_string(desp = 'gpu_info')
- gpu_table_query = """
- SELECT {schema_madlib}.gpu_configuration('{gpu_info_table}')
- """.format(**locals())
- plpy.execute(gpu_table_query)
- gpu_query = """
- SELECT hostname, count(*) AS count FROM {gpu_info_table} GROUP BY hostname
- """.format(**locals())
- gpu_query_result = plpy.execute(gpu_query)
- plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
- if not gpu_query_result:
- plpy.error("{0} error: No GPUs configured on hosts.".format(module_name))
-
- host_dict = {}
- for i in gpu_query_result:
- host_dict[i['hostname']] = int(i['count'])
-
- seg_query = """
- SELECT hostname, content AS segment_id
- FROM gp_segment_configuration
- WHERE content != -1 AND role = 'p'
- """
- seg_query_result = plpy.execute(seg_query)
-
- accessible_gpus_for_seg = [0] * len(seg_query_result)
- warning_flag = True
- for i in seg_query_result:
- if i['hostname'] in host_dict.keys():
- accessible_gpus_for_seg[i['segment_id']] = host_dict[i['hostname']]
- if 0 < accessible_gpus_for_seg[i['segment_id']] < segments_per_host and warning_flag:
- plpy.warning(
- 'The number of GPUs per segment host is less than the number of '
- 'segments per segment host. When different segments share the '
- 'same GPU, this may fail in some scenarios. The current '
- 'recommended configuration is to have 1 GPU available per segment.')
- warning_flag = False
- return accessible_gpus_for_seg
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
deleted file mode 100644
index f3b02c7..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
+++ /dev/null
@@ -1,192 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-
-import plpy
-from collections import OrderedDict
-from madlib_keras_validator import MstLoaderInputValidator
-from utilities.control import MinWarning
-from utilities.utilities import add_postfix
-from madlib_keras_wrapper import convert_string_of_args_to_dict
-from keras_model_arch_table import ModelArchSchema
-
-class ModelSelectionSchema:
- MST_KEY = 'mst_key'
- MODEL_ID = ModelArchSchema.MODEL_ID
- MODEL_ARCH_TABLE = 'model_arch_table'
- COMPILE_PARAMS = 'compile_params'
- FIT_PARAMS = 'fit_params'
- col_types = ('SERIAL', 'INTEGER', 'VARCHAR', 'VARCHAR')
-
-@MinWarning("warning")
-class MstLoader():
- """The utility class for loading a model selection table with model parameters.
-
- Currently just takes all combinations of input parameters passed. This
- utility validates the inputs.
-
- Attributes:
- compile_params_list (list): The input list of compile params choices.
- fit_params_list (list): The input list of fit params choices.
- model_id_list (list): The input list of model id choices.
- model_arch_table (str): The name of model architechure table.
- model_selection_table (str): The name of the output mst table.
- msts (list): The list of generated msts.
-
- """
-
- def __init__(self,
- model_arch_table,
- model_selection_table,
- model_id_list,
- compile_params_list,
- fit_params_list,
- **kwargs):
-
- self.model_arch_table = model_arch_table
- self.model_selection_table = model_selection_table
- self.model_selection_summary_table = add_postfix(
- model_selection_table, "_summary")
- self.model_id_list = sorted(list(set(model_id_list)))
- MstLoaderInputValidator(
- model_arch_table=self.model_arch_table,
- model_selection_table=self.model_selection_table,
- model_selection_summary_table=self.model_selection_summary_table,
- model_id_list=self.model_id_list,
- compile_params_list=compile_params_list,
- fit_params_list=fit_params_list
- )
- self.compile_params_list = self.params_preprocessed(
- compile_params_list)
- self.fit_params_list = self.params_preprocessed(fit_params_list)
-
- self.msts = []
-
- self.find_combinations()
-
- def load(self):
- """The entry point for loading the model selection table.
- """
- # All of the side effects happen in this function.
- self.create_mst_table()
- self.create_mst_summary_table()
- self.insert_into_mst_table()
-
- def params_preprocessed(self, list_strs):
- """Preprocess the input lists. Eliminate white spaces and sort them.
-
- Args:
- list_strs (list): A list of strings.
-
- Returns:
- list: The preprocessed list of strings.
- """
-
- dict_dedup = {}
- for string in list_strs:
- d = convert_string_of_args_to_dict(string)
- hash_tuple = tuple( '{0} = {1}'\
- .format(x, d[x]) for x in sorted(d.keys()))
- dict_dedup[hash_tuple] = string
-
- return dict_dedup.values()
-
- def find_combinations(self):
- """Backtracking helper for generating the combinations.
- """
- param_grid = OrderedDict([
- (ModelSelectionSchema.MODEL_ID, self.model_id_list),
- (ModelSelectionSchema.COMPILE_PARAMS, self.compile_params_list),
- (ModelSelectionSchema.FIT_PARAMS, self.fit_params_list)
- ])
-
- def find_combinations_helper(msts, p, i):
- param_names = param_grid.keys()
- if i < len(param_names):
- for x in param_grid[param_names[i]]:
- p[param_names[i]] = x
- find_combinations_helper(msts, p, i + 1)
- else:
- msts.append(p.copy())
- find_combinations_helper(self.msts, {}, 0)
-
- def create_mst_table(self):
- """Initialize the output mst table.
- """
- create_query = """
- CREATE TABLE {self.model_selection_table} (
- {mst_key} SERIAL,
- {model_id} INTEGER,
- {compile_params} VARCHAR,
- {fit_params} VARCHAR,
- unique ({model_id}, {compile_params}, {fit_params})
- );
- """.format(self=self,
- mst_key=ModelSelectionSchema.MST_KEY,
- model_id=ModelSelectionSchema.MODEL_ID,
- compile_params=ModelSelectionSchema.COMPILE_PARAMS,
- fit_params=ModelSelectionSchema.FIT_PARAMS)
- with MinWarning('warning'):
- plpy.execute(create_query)
-
- def create_mst_summary_table(self):
- """Initialize the output mst table.
- """
- create_query = """
- CREATE TABLE {self.model_selection_summary_table} (
- {model_arch_table} VARCHAR
- );
- """.format(self=self,
- model_arch_table=ModelSelectionSchema.MODEL_ARCH_TABLE)
- with MinWarning('warning'):
- plpy.execute(create_query)
-
- def insert_into_mst_table(self):
- """Insert every thing in self.msts into the mst table.
- """
- for mst in self.msts:
- model_id = mst[ModelSelectionSchema.MODEL_ID]
- compile_params = mst[ModelSelectionSchema.COMPILE_PARAMS]
- fit_params = mst[ModelSelectionSchema.FIT_PARAMS]
- insert_query = """
- INSERT INTO
- {self.model_selection_table}(
- {model_id_col},
- {compile_params_col},
- {fit_params_col}
- )
- VALUES (
- {model_id},
- $${compile_params}$$,
- $${fit_params}$$
- )
- """.format(model_id_col=ModelSelectionSchema.MODEL_ID,
- compile_params_col=ModelSelectionSchema.COMPILE_PARAMS,
- fit_params_col=ModelSelectionSchema.FIT_PARAMS,
- **locals())
- plpy.execute(insert_query)
- insert_summary_query = """
- INSERT INTO
- {self.model_selection_summary_table}(
- {model_arch_table_name}
- )
- VALUES (
- $${self.model_arch_table}$$
- )
- """.format(model_arch_table_name=ModelSelectionSchema.MODEL_ARCH_TABLE,
- **locals())
- plpy.execute(insert_summary_query)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in
deleted file mode 100644
index c15757c..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in
+++ /dev/null
@@ -1,437 +0,0 @@
-/* ----------------------------------------------------------------------- *//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *
- * @file madlib_keras_model_selection.sql_in
- *
- * @brief SQL functions for model hopper distributed training
- * @date August 2019
- *
- *
- *//* ----------------------------------------------------------------------- */
-
-m4_include(`SQLCommon.m4')
-/**
-@addtogroup grp_keras_setup_model_selection
-
-@brief Utility function to set up a model selection table for model architecture search
-and hyperparameter tuning.
-
-\warning <em> This MADlib method is still in early stage development.
-Interface and implementation are subject to change. </em>
-
-<div class="toc"><b>Contents</b><ul>
-<li class="level1"><a href="#load_mst_table">Load Model Selection Table</a></li>
-<li class="level1"><a href="#example">Examples</a></li>
-<li class="level1"><a href="#notes">Notes</a></li>
-<li class="level1"><a href="#related">Related Topics</a></li>
-</ul></div>
-
-This utility function sets up a model selection table
-for use by the multiple model Keras fit feature of MADlib.
-By model selection we mean both hyperparameter tuning and
-model architecture search. The table defines the unique combinations
-of model architectures, compile and fit parameters
-to run on a massively parallel processing database cluster.
-
-@anchor load_mst_table
-@par Load Model Selection Table
-
-<pre class="syntax">
-load_model_selection_table(
- model_arch_table,
- model_selection_table,
- model_id_list,
- compile_params_list,
- fit_params_list
- )
-</pre>
-
-\b Arguments
-<dl class="arglist">
- <dt>model_arch_table</dt>
- <dd>VARCHAR. Table containing model architectures and weights.
- For more information on this table
- refer to <a href="group__grp__keras__model__arch.html">Load Model</a>.
- </dd>
-
- <dt>model_selection_table</dt>
- <dd>VARCHAR. Model selection table created by this utility. A summary table
- named <model_selection_table>_summary is also created. Contents of both output
- tables are described below.
- </dd>
-
- <dt>model_id_list</dt>
- <dd>INTEGER[]. Array of model IDs from the 'model_arch_table' to be included
- in the run combinations. For hyperparameter search, this will typically be
- one model ID. For model architecture search, this will be the different model IDs
- that you want to test.
- </dd>
-
- <dt>compile_params_list</dt>
- <dd>VARCHAR[]. Array of compile parameters to be tested. Each element
- of the array should consist of a string of compile parameters
- exactly as it is to be passed to Keras.
- </dd>
-
- <dt>fit_params_list</dt>
- <dd>VARCHAR[]. Array of fit parameters to be tested. Each element
- of the array should consist of a string of fit parameters
- exactly as it is to be passed to Keras.
- </dd>
-
-</dl>
-
-<b>Output table</b>
-<br>
- The model selection output table contains the following columns:
- <table class="output">
- <tr>
- <th>mst_key</th>
- <td>INTEGER. ID that defines a unique tuple for
- model architecture-compile parameters-fit parameters.
- </td>
- </tr>
- <tr>
- <th>model_id</th>
- <td>VARCHAR. Model architecture ID from the 'model_arch_table'.
- </td>
- </tr>
- <tr>
- <th>compile_params</th>
- <td>VARCHAR. Keras compile parameters.
- </td>
- </tr>
- <tr>
- <th>fit_params</th>
- <td>VARCHAR. Keras fit parameters.
- </td>
- </tr>
- </table>
- A summary table named <model_selection_table>_summary is
- also created, which contains the following column:
- <table class="output">
- <tr>
- <th>model_arch_table</th>
- <td>VARCHAR. Name of the model architecture table containing the
- model architecture IDs.
- </td>
- </tr>
- </table>
-</br>
-
-@anchor example
-@par Examples
--# The model selection table works in conjunction with a model architecture table,
-so we first create a model architecture table with two different models. Use Keras to define
-a model architecture with 1 hidden layer:
-<pre class="example">
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model1 = Sequential()
-model1.add(Dense(10, activation='relu', input_shape=(4,)))
-model1.add(Dense(10, activation='relu'))
-model1.add(Dense(3, activation='softmax'))
-model1.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type) Output Shape Param #
-=================================================================
-dense_1 (Dense) (None, 10) 50
-_________________________________________________________________
-dense_2 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_3 (Dense) (None, 3) 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-</pre>
-Export the model to JSON:
-<pre class="example">
-model1.to_json()
-</pre>
-<pre class="result">
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-</pre>
-Now use Keras to define
-a model architecture with 2 hidden layers:
-<pre class="example">
-model2 = Sequential()
-model2.add(Dense(10, activation='relu', input_shape=(4,)))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(3, activation='softmax'))
-model2.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type) Output Shape Param #
-=================================================================
-dense_4 (Dense) (None, 10) 50
-_________________________________________________________________
-dense_5 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_6 (Dense) (None, 10) 110
-_________________________________________________________________
-dense_7 (Dense) (None, 3) 33
-=================================================================
-Total params: 303
-Trainable params: 303
-Non-trainable params: 0
-\endverbatim
-</pre>
-Export the model to JSON:
-<pre class="example">
-model2.to_json()
-</pre>
-<pre class="result">
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-</pre>
-
--# Load both models into the architecture table:
-<pre class="example">
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Sophie', -- Name
- 'MLP with 1 hidden layer' -- Descr
-);
-SELECT madlib.load_keras_model('model_arch_library', -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json, -- JSON blob
- NULL, -- Weights
- 'Maria', -- Name
- 'MLP with 2 hidden layers' -- Descr
-);
-SELECT model_id, name, description FROM model_arch_library ORDER BY model_id;
-</pre>
-<pre class="result">
- model_id | name | description
-----------+--------+--------------------------
- 1 | Sophie | MLP with 1 hidden layer
- 2 | Maria | MLP with 2 hidden layers
-(2 rows)
-</pre>
-
--# Load model selection table. Select the model(s) from the model
-architecture table that you want to run, along with the compile and
-fit parameters. Unique combinations will be created:
-<pre class="example">
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT madlib.load_model_selection_table('model_arch_library', -- model architecture table
- 'mst_table', -- model selection table output
- ARRAY[1,2], -- model ids from model architecture table
- ARRAY[ -- compile params
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
- ],
- ARRAY[ -- fit params
- $$batch_size=4,epochs=1$$,
- $$batch_size=8,epochs=1$$
- ]
- );
-SELECT * FROM mst_table ORDER BY mst_key;
-</pre>
-<pre class="result">
- mst_key | model_id | compile_params | fit_params
----------+---------------+---------------------------------------------------------------------------------+-----------------------
- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
- 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
- 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
- 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
- 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
- 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
- 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
- 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
- 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
- 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-(12 rows)
-</pre>
-The name of the model architecture table is stored in the summary table:
-<pre class="example">
-SELECT * FROM mst_table_summary;
-</pre>
-<pre class="result">
- model_arch_table
---------------------+
- model_arch_library
-(1 row)
-</pre>
-
--# Create model selection table manually. If you would like to
-have more control over the set of model selection parameters
-to run, you can manually create the model selection table and
-the associated summary table. Both must be created since
-they are needed by the multiple model fit module.
-For example, let's say we don't want all combinations
-but only want batch_size=4 for model_id=1 and batch_size=8 for model_id=2:
-<pre class="example">
-DROP TABLE IF EXISTS mst_table_manual;
-CREATE TABLE mst_table_manual(
- mst_key serial,
- model_id integer,
- compile_params varchar,
- fit_params varchar
-);
-INSERT INTO mst_table_manual(model_id, compile_params, fit_params) VALUES
-(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
-(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
-(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
-(2, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, 'batch_size=8,epochs=1'),
-(2, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy']$$, 'batch_size=8,epochs=1'),
-(2, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$, 'batch_size=8,epochs=1');
-SELECT * FROM mst_table_manual ORDER BY mst_key;
-</pre>
-<pre class="result">
- mst_key | model_id | compile_params | fit_params
----------+---------------+---------------------------------------------------------------------------------+-----------------------
- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
- 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
- 3 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 4 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
- 5 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
- 6 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-(6 rows)
-</pre>
-Create the summary table which must be named
-with the model selection output table appended by "_summary":
-<pre class="example">
-DROP TABLE IF EXISTS mst_table_manual_summary;
-CREATE TABLE mst_table_manual_summary (
- model_arch_table varchar
-);
-INSERT INTO mst_table_manual_summary(model_arch_table) VALUES
-('model_arch_library');
-SELECT * FROM mst_table_manual_summary;
-</pre>
-<pre class="result">
- model_arch_table
---------------------+
- model_arch_library
-(1 row)
-</pre>
-
--# Generate hyperparameters automatically. You can use other
-libraries or methods to generate hyperparameters according
-to the tests that you want to run. For example, let's randomly
-generate batch size from powers of 2 and learning
-rate on a log scale.
-We use psycopg which is a PostgreSQL database adapter
-for the Python programming language.
-<pre class="example">
-import numpy as np
-import psycopg2 as p2
-conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')
-#conn = p2.connect('postgresql://fmcquillan@localhost:5432/madlib')
-cur = conn.cursor()
-\#
-%sql DROP TABLE IF EXISTS mst_table_auto, mst_table_auto_summary;
-\#
-\#compile params
-learning_rate = np.random.permutation([0.1,0.01,0.001,0.0001])[:3]
-compile_param1 = "loss='categorical_crossentropy',optimizer='Adam(lr=" + str(learning_rate[0]) + ")',metrics=['accuracy']"
-compile_param2 = "loss='categorical_crossentropy',optimizer='Adam(lr=" + str(learning_rate[1]) + ")',metrics=['accuracy']"
-compile_param3 = "loss='categorical_crossentropy',optimizer='Adam(lr=" + str(learning_rate[2]) + ")',metrics=['accuracy']"
-compile_params = [compile_param1,compile_param2,compile_param3]
-\#
-\#fit params
-batch_size = np.random.permutation([4,8,16,32,64])[:2]
-fit_param1 = "batch_size=" + str(batch_size[0]) + ",epochs=1"
-fit_param2 = "batch_size=" + str(batch_size[1]) + ",epochs=1"
-fit_params = [fit_param1,fit_param2]
-\#
-query = "SELECT madlib.load_model_selection_table('model_arch_library', 'mst_table_auto', ARRAY[1,2], %s, %s);"
-\#
-cur.execute(query,[compile_params, fit_params])
-conn.commit()
-\#
-\# review model selection table
-%sql SELECT * FROM mst_table_auto ORDER BY mst_key;
-</pre>
-<pre class="result">
- mst_key | model_id | compile_params | fit_params
----------+---------------+----------------------------------------------------------------------------------+-----------------------
- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=8,epochs=1
- 3 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
- 4 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
- 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
- 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=8,epochs=1
- 9 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
- 10 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
- 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
- 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-(12 rows)
-</pre>
-The name of the model architecture table is stored in the summary table:
-<pre class="example">
-SELECT * FROM mst_table_auto_summary;
-</pre>
-<pre class="result">
- model_arch_table
---------------------+
- model_arch_library
-(1 row)
-</pre>
-
-@anchor notes
-@par Notes
-
-1. In this method, the same compile and fit parameters are applied to all model architectures
-when generating combinations. However, you may wish to have different compile and fit parameters
-for each model. To do so, call 'load_model_selection_table'
-multiple times - once for each model. Then you can combine the resulting tables using UNION or other means.
-Note that the 'mst_key' must be unique so you will need to regenerate it in your final combined table.
-
-@anchor related
-@par Related Topics
-
-See keras_model_arch_table.sql_in
-
-*/
-
-CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_model_selection_table(
- model_arch_table VARCHAR,
- model_selection_table VARCHAR,
- model_id_list INTEGER[],
- compile_params_list VARCHAR[],
- fit_params_list VARCHAR[]
-) RETURNS VOID AS $$
- PythonFunctionBodyOnly(`deep_learning', `madlib_keras_model_selection')
- with AOControl(False):
- mst_loader = madlib_keras_model_selection.MstLoader(**globals())
- mst_loader.load()
-$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
deleted file mode 100644
index 412e63b..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
+++ /dev/null
@@ -1,456 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import plpy
-
-import keras
-from keras import backend as K
-from keras.layers import *
-from keras.models import *
-from keras.optimizers import *
-
-from model_arch_info import *
-from madlib_keras_helper import *
-from madlib_keras_validator import *
-from predict_input_params import PredictParamsProcessor
-from utilities.control import MinWarning
-from utilities.utilities import _assert
-from utilities.utilities import add_postfix
-from utilities.utilities import create_cols_from_array_sql_string
-from utilities.utilities import get_segments_per_host
-from utilities.utilities import unique_string
-from utilities.validate_args import get_expr_type
-from utilities.validate_args import input_tbl_valid
-
-from madlib_keras_wrapper import *
-
-class BasePredict():
- def __init__(self, schema_madlib, table_to_validate, test_table, id_col,
- independent_varname, output_table, pred_type, use_gpus, module_name):
- self.schema_madlib = schema_madlib
- self.table_to_validate = table_to_validate
- self.test_table = test_table
- self.id_col = id_col
- self.independent_varname = independent_varname
- self.output_table = output_table
- self.pred_type = pred_type
- self.module_name = module_name
-
- self.segments_per_host = get_segments_per_host()
- self.use_gpus = use_gpus if use_gpus else False
- if self.use_gpus:
- accessible_gpus_for_seg = get_accessible_gpus_for_seg(schema_madlib, self.segments_per_host, self.module_name)
- _assert(len(set(accessible_gpus_for_seg)) == 1,
- '{0}: Asymmetric gpu configurations are not supported'.format(self.module_name))
- self.gpus_per_host = accessible_gpus_for_seg[0]
- else:
- self.gpus_per_host = 0
-
- self._set_default_pred_type()
-
- def _set_default_pred_type(self):
- self.pred_type = 'response' if not self.pred_type else self.pred_type
- self.is_response = True if self.pred_type == 'response' else False
-
- def call_internal_keras(self):
- if self.is_response:
- pred_col_name = add_postfix("estimated_", self.dependent_varname)
- pred_col_type = self.dependent_vartype
- else:
- pred_col_name = "prob"
- pred_col_type = 'double precision'
-
- intermediate_col = unique_string()
- class_values = strip_trailing_nulls_from_class_values(self.class_values)
-
- prediction_select_clause, create_table_columns = create_cols_from_array_sql_string(
- class_values, intermediate_col, pred_col_name,
- pred_col_type, self.is_response, self.module_name)
- gp_segment_id_col, seg_ids_test, \
- images_per_seg_test = get_image_count_per_seg_for_non_minibatched_data_from_db(
- self.test_table)
- segments_per_host = get_segments_per_host()
-
- select_segmentid_comma = ""
- group_by_clause = ""
- join_cond_on_segmentid = ""
- if not is_platform_pg():
- select_segmentid_comma = "{self.test_table}.gp_segment_id AS gp_segment_id,".format(self=self)
- group_by_clause = "GROUP BY {self.test_table}.gp_segment_id".format(self=self)
- join_cond_on_segmentid = "{self.test_table}.gp_segment_id=min_ctid.gp_segment_id AND".format(self=self)
-
- # Calling CREATE TABLE instead of CTAS, to ensure that the plan_cache_mode
- # guc codepath is called when passing in the weights
- plpy.execute("""
- CREATE TABLE {self.output_table}
- ({self.id_col} {self.id_col_type}, {create_table_columns})
- """.format(self=self, create_table_columns=create_table_columns))
- # Passing huge model weights to internal_keras_predict() for each row
- # resulted in slowness of overall madlib_keras_predict().
- # To avoid this, a CASE is added to pass the model weights only for
- # the very first row(min(ctid)) that is fetched on each segment and NULL
- # for the other rows.
- predict_query = plpy.prepare("""
- INSERT INTO {self.output_table}
- SELECT {self.id_col}::{self.id_col_type}, {prediction_select_clause}
- FROM (
- SELECT {self.test_table}.{self.id_col},
- ({self.schema_madlib}.internal_keras_predict
- ({self.independent_varname},
- $1,
- CASE WHEN {self.test_table}.ctid = min_ctid.ctid THEN $2 ELSE NULL END,
- {self.is_response},
- {self.normalizing_const},
- {gp_segment_id_col},
- ARRAY{seg_ids_test},
- ARRAY{images_per_seg_test},
- {self.use_gpus},
- {self.gpus_per_host},
- {segments_per_host})
- ) AS {intermediate_col}
- FROM {self.test_table}
- LEFT JOIN
- (SELECT {select_segmentid_comma} MIN({self.test_table}.ctid) AS ctid
- FROM {self.test_table}
- {group_by_clause}) min_ctid
- ON {join_cond_on_segmentid} {self.test_table}.ctid=min_ctid.ctid
- ) q
- """.format(self=self, prediction_select_clause=prediction_select_clause,
- seg_ids_test=seg_ids_test,
- images_per_seg_test=images_per_seg_test,
- gp_segment_id_col=gp_segment_id_col,
- segments_per_host=segments_per_host,
- intermediate_col=intermediate_col,
- select_segmentid_comma=select_segmentid_comma,
- group_by_clause=group_by_clause,
- join_cond_on_segmentid=join_cond_on_segmentid),
- ["text", "bytea"])
- plpy.execute(predict_query, [self.model_arch, self.model_weights])
-
- def set_default_class_values(self, class_values):
- self.class_values = class_values
- if self.pred_type == 'prob':
- return
- if self.class_values is None:
- num_classes = get_num_classes(self.model_arch)
- self.class_values = range(0, num_classes)
-
-@MinWarning("warning")
-class Predict(BasePredict):
- def __init__(self, schema_madlib, model_table,
- test_table, id_col, independent_varname,
- output_table, pred_type, use_gpus,
- mst_key, **kwargs):
-
- self.module_name = 'madlib_keras_predict'
- self.model_table = model_table
- self.mst_key = mst_key
- self.is_mult_model = mst_key is not None
- if self.model_table:
- self.model_summary_table = add_postfix(self.model_table, "_summary")
-
- BasePredict.__init__(self, schema_madlib, model_table, test_table,
- id_col, independent_varname,
- output_table, pred_type,
- use_gpus, self.module_name)
- param_proc = PredictParamsProcessor(self.model_table, self.module_name, self.mst_key)
- if self.is_mult_model:
- self.temp_summary_view = param_proc.model_summary_table
- self.model_summary_table = self.temp_summary_view
- self.dependent_vartype = param_proc.get_dependent_vartype()
- self.model_weights = param_proc.get_model_weights()
- self.model_arch = param_proc.get_model_arch()
- class_values = param_proc.get_class_values()
- self.set_default_class_values(class_values)
- self.normalizing_const = param_proc.get_normalizing_const()
- self.dependent_varname = param_proc.get_dependent_varname()
-
- self.validate()
- self.id_col_type = get_expr_type(self.id_col, self.test_table)
- BasePredict.call_internal_keras(self)
- if self.is_mult_model:
- plpy.execute("DROP VIEW IF EXISTS {}".format(self.temp_summary_view))
-
- def validate(self):
- input_tbl_valid(self.model_table, self.module_name)
- if self.is_mult_model and not columns_exist_in_table(self.model_table, ['mst_key']):
- plpy.error("{self.module_name}: Single model should not pass mst_key".format(**locals()))
- if not self.is_mult_model and columns_exist_in_table(self.model_table, ['mst_key']):
- plpy.error("{self.module_name}: Multi-model needs to pass mst_key".format(**locals()))
- InputValidator.validate_predict_evaluate_tables(
- self.module_name, self.model_table, self.model_summary_table,
- self.test_table, self.output_table, self.independent_varname)
-
- InputValidator.validate_id_in_test_tbl(
- self.module_name, self.test_table, self.id_col)
-
- InputValidator.validate_class_values(
- self.module_name, self.class_values, self.pred_type, self.model_arch)
- input_shape = get_input_shape(self.model_arch)
- InputValidator.validate_pred_type(
- self.module_name, self.pred_type, self.class_values)
- InputValidator.validate_input_shape(
- self.test_table, self.independent_varname, input_shape, 1)
-
-@MinWarning("warning")
-class PredictBYOM(BasePredict):
- def __init__(self, schema_madlib, model_arch_table, model_id,
- test_table, id_col, independent_varname, output_table,
- pred_type, use_gpus, class_values, normalizing_const,
- **kwargs):
-
- self.module_name='madlib_keras_predict_byom'
- self.model_arch_table = model_arch_table
- self.model_id = model_id
- self.class_values = class_values
- self.normalizing_const = normalizing_const
- self.dependent_varname = 'dependent_var'
- BasePredict.__init__(self, schema_madlib, model_arch_table,
- test_table, id_col, independent_varname,
- output_table, pred_type, use_gpus, self.module_name)
- if self.is_response:
- self.dependent_vartype = 'text'
- else:
- self.dependent_vartype = 'double precision'
- ## Set default values for norm const and class_values
- # use_gpus and pred_type are defaulted in base_predict's init
- self.normalizing_const = normalizing_const
- if self.normalizing_const is None:
- self.normalizing_const = DEFAULT_NORMALIZING_CONST
- InputValidator.validate_predict_byom_tables(
- self.module_name, self.model_arch_table, self.model_id,
- self.test_table, self.id_col, self.output_table,
- self.independent_varname)
- self.validate_and_set_defaults()
- self.id_col_type = get_expr_type(self.id_col, self.test_table)
- BasePredict.call_internal_keras(self)
-
- def validate_and_set_defaults(self):
- # Set some defaults first and then validate and then set some more defaults
- self.model_arch, self.model_weights = get_model_arch_weights(
- self.model_arch_table, self.model_id)
- # Assert model_weights and model_arch are not empty.
- _assert(self.model_weights and self.model_arch,
- "{0}: Model weights and architecture should not be NULL.".format(
- self.module_name))
- self.set_default_class_values(self.class_values)
-
- InputValidator.validate_pred_type(
- self.module_name, self.pred_type, self.class_values)
- InputValidator.validate_normalizing_const(
- self.module_name, self.normalizing_const)
- InputValidator.validate_class_values(
- self.module_name, self.class_values, self.pred_type, self.model_arch)
- InputValidator.validate_input_shape(
- self.test_table, self.independent_varname,
- get_input_shape(self.model_arch), 1)
-
-def internal_keras_predict(independent_var, model_architecture, model_weights,
- is_response, normalizing_const, current_seg_id, seg_ids,
- images_per_seg, use_gpus, gpus_per_host, segments_per_host,
- **kwargs):
- SD = kwargs['SD']
- model_key = 'segment_model_predict'
- row_count_key = 'row_count'
- try:
- device_name = get_device_name_and_set_cuda_env( gpus_per_host, current_seg_id)
- if model_key not in SD:
- set_keras_session(device_name, gpus_per_host, segments_per_host)
- model = model_from_json(model_architecture)
- set_model_weights(model, model_weights)
- SD[model_key] = model
- SD[row_count_key] = 0
- else:
- model = SD[model_key]
- SD[row_count_key] += 1
-
- # Since the test data isn't mini-batched,
- # we have to make sure that the test data np array has the same
- # number of dimensions as input_shape. So we add a dimension to x.
- independent_var = expand_input_dims(independent_var)
- independent_var /= normalizing_const
-
- if is_response:
- with K.tf.device(device_name):
- y_prob = model.predict(independent_var)
- proba_argmax = y_prob.argmax(axis=-1)
- # proba_argmax is a list with exactly one element in it. That element
- # refers to the index containing the largest probability value in the
- # output of Keras' predict function.
- result = proba_argmax
- else:
- with K.tf.device(device_name):
- probs = model.predict(independent_var)
- # probs is a list containing a list of probability values, of all
- # class levels. Since we are assuming each input is a single image,
- # and not mini-batched, this list contains exactly one list in it,
- # so return back the first list in probs.
- result = probs[0]
- total_images = get_image_count_per_seg_from_array(seg_ids.index(current_seg_id),
- images_per_seg)
-
- if SD[row_count_key] == total_images:
- SD.pop(model_key, None)
- SD.pop(row_count_key, None)
- clear_keras_session()
- return result
- except Exception as ex:
- SD.pop(model_key, None)
- SD.pop(row_count_key, None)
- clear_keras_session()
- plpy.error(ex)
-
-
-def predict_help(schema_madlib, message, **kwargs):
- """
- Help function for keras predict
-
- Args:
- @param schema_madlib
- @param message: string, Help message string
- @param kwargs
-
- Returns:
- String. Help/usage information
- """
- if not message:
- help_string = """
------------------------------------------------------------------------
- SUMMARY
------------------------------------------------------------------------
-This function allows the user to predict using a madlib_keras_fit trained
-model.
-
-For more details on function usage:
- SELECT {schema_madlib}.madlib_keras_predict('usage')
- """
- elif message in ['usage', 'help', '?']:
- help_string = """
------------------------------------------------------------------------
- USAGE
------------------------------------------------------------------------
- SELECT {schema_madlib}.madlib_keras_predict(
- model_table, -- Name of the table containing the model
- test_table, -- Name of the table containing the evaluation dataset
- id_col, -- Name of the id column in the test data table
- independent_varname, -- Name of the column with independent
- variables in the test table
- output_table, -- Name of the output table
- pred_type, -- The type of the desired output
- use_gpus, -- Flag for enabling GPU support
- mst_key -- Identifier for the desired model out of multimodel
- training output
- )
- );
-
------------------------------------------------------------------------
- OUTPUT
------------------------------------------------------------------------
-The output table ('output_table' above) contains the following columns:
-
-id: Gives the 'id' for each prediction, corresponding
- to each row from the test_table.
-estimated_COL_NAME: (For pred_type='response') The estimated class for
- classification, where COL_NAME is the name of the
- column to be predicted from test data.
-prob_CLASS: (For pred_type='prob' for classification) The
- probability of a given class. There will be one
- column for each class in the training data.
- TODO change this
-"""
- else:
- help_string = "No such option. Use {schema_madlib}.madlib_keras_predict()"
-
- return help_string.format(schema_madlib=schema_madlib)
-
-def predict_byom_help(schema_madlib, message, **kwargs):
- """
- Help function for keras predict
-
- Args:
- @param schema_madlib
- @param message: string, Help message string
- @param kwargs
-
- Returns:
- String. Help/usage information
- """
- if not message:
- help_string = """
------------------------------------------------------------------------
- SUMMARY
------------------------------------------------------------------------
-This function allows the user to predict with their own pre trained model (note
-that this model doesn't have to be trained using MADlib.)
-
-For more details on function usage:
- SELECT {schema_madlib}.madlib_keras_predict_byom('usage')
- """
- elif message in ['usage', 'help', '?']:
- help_string = """
------------------------------------------------------------------------
- USAGE
------------------------------------------------------------------------
- SELECT {schema_madlib}.madlib_keras_predict_byom(
- model_arch_table, -- Name of the table containing the model architecture
- and the pre trained model weights
- model_id, -- This is the id in 'model_arch_table' containing the
- model architecture
- test_table, -- Name of the table containing the evaluation dataset
- id_col, -- Name of the id column in the test data table
- independent_varname, -- Name of the column with independent
- variables in the test table
- output_table, -- Name of the output table
- pred_type, -- The type of the desired output
- use_gpus, -- Flag for enabling GPU support
- class_values, -- List of class labels that were used while training the
- model. If class_values is passed in as NULL, the output
- table will have a column named 'prob' which is an array
- of probabilities of all the classes.
- Otherwise if class_values is not NULL, then the output
- table will contain a column for each class/label from
- the training data
- normalizing_const, -- Normalizing constant used for standardizing arrays in
- independent_varname
- )
- );
-
------------------------------------------------------------------------
- OUTPUT
------------------------------------------------------------------------
-The output table ('output_table' above) contains the following columns:
-
-id: Gives the 'id' for each prediction, corresponding
- to each row from the test_table.
-estimated_dependent_var: (For pred_type='response') The estimated class for
- classification. If class_values is passed in as NULL, then we
- assume that the class labels are [0,1,2...,n] where n in the
- num of classes in the model architecture.
-prob_CLASS: (For pred_type='prob' for classification) The
- probability of a given class.
- If class_values is passed in as NULL, we create just one column
- called 'prob' which is an array of probabilites of all the classes
- Otherwise if class_values is not NULL, then there will be one
- column for each class in the training data.
-"""
- else:
- help_string = "No such option. Use {schema_madlib}.madlib_keras_predict_byom()"
-
- return help_string.format(schema_madlib=schema_madlib)
-# ---------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_serializer.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_serializer.py_in
deleted file mode 100644
index 6fa210c..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_serializer.py_in
+++ /dev/null
@@ -1,162 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import numpy as np
-from utilities.utilities import _assert
-
-# TODO
-# 1. Current serializing logic
- # serialized string -> byte string
- # np.array(np.array(image_count).concatenate(weights_np_array)).tostring()
- # Proposed logic
- # image_count can be a separate value
- # weights -> np.array(weights).tostring()
- # combine these 2 into one string by a random splitter
- # serialized string -> imagecount_splitter_weights
-# 2. combine the serialize_state_with_nd_weights and serialize_state_with_1d_weights
- # into one function called serialize_state. This function can infer the shape
- # of the model weights and then flatten if they are nd weights.
-# 3. Same as 2 for deserialize
-
-
-"""
-workflow
-1. Set initial weights in madlib keras fit function.
-2. Serialize these initial model weights as a byte string and pass it to keras step
-3. Deserialize the state passed from the previous step into a list of nd weights
-that will be passed on to model.set_weights()
-4. At the end of each buffer in fit transition, serialize the image count and
-the model weights into a bytestring that will be passed on to the fit merge function.
-5. In fit merge, deserialize the state as image and 1d np arrays. Do some averaging
-operations and serialize them again into a state which contains the image
-and the 1d state. same for fit final
-6. Return the final state from fit final to fit which will then be deserialized
-as 1d weights to be passed on to the evaluate function
-"""
-def get_serialized_1d_weights_from_state(state):
- """
- Output of this function is used to deserialize the output of each iteration
- of madlib keras step UDA.
-
- :param state: bytestring serialized model state containing image count
- and weights
- :return: model weights serialized as bytestring
- """
- _ , weights = deserialize_as_image_1d_weights(state)
- return weights.tostring()
-
-def serialize_state_with_nd_weights(image_count, model_weights):
- """
- This function is called when the output of keras.get_weights() (list of nd
- np arrays) has to be converted into a serialized model state.
-
- :param image_count: float value
- :param model_weights: a list of numpy arrays, what you get from
- keras.get_weights()
- :return: Image count and model weights serialized into a bytestring format
-
- """
- if model_weights is None:
- return None
- flattened_weights = [w.flatten() for w in model_weights]
- state = [np.array([image_count])] + flattened_weights
- state = np.concatenate(state)
- return np.float32(state).tostring()
-
-
-def serialize_state_with_1d_weights(image_count, model_weights):
- """
- This function is called when the weights are to be passed to the keras fit
- merge and final functions.
-
- :param image_count: float value
- :param model_weights: a single flattened numpy array containing all of the
- weights
- :return: Image count and model weights serialized into a bytestring format
-
- """
- if model_weights is None:
- return None
- merge_state = np.array([image_count])
- merge_state = np.concatenate((merge_state, model_weights))
- merge_state = np.float32(merge_state)
- return merge_state.tostring()
-
-
-def deserialize_as_image_1d_weights(state):
- """
- This function is called when the model state needs to be deserialized in
- the keras fit merge and final functions.
-
- :param state: the stringified (serialized) state containing image_count and
- model_weights
- :return:
- image_count: total buffer counts processed
- model_weights: a single flattened numpy array containing all of the
- weights
- """
- if not state:
- return None
- state = np.fromstring(state, dtype=np.float32)
- return float(state[0]), state[1:]
-
-
-def serialize_nd_weights(model_weights):
- """
- This function is called for passing the initial model weights from the keras
- fit function to the keras fit transition function.
- :param model_weights: a list of numpy arrays, what you get from
- keras.get_weights()
- :return: Model weights serialized into a bytestring format
- """
- if model_weights is None:
- return None
- flattened_weights = [w.flatten() for w in model_weights]
- flattened_weights = np.concatenate(flattened_weights)
- return np.float32(flattened_weights).tostring()
-
-
-def deserialize_as_nd_weights(model_weights_serialized, model_shapes):
- """
- The output of this function is used to set keras model weights using the
- function model.set_weights()
- :param model_weights_serialized: bytestring containing model weights
- :param model_shapes: list containing the shapes of each layer.
- :return: list of nd numpy arrays containing all of the
- weights
- """
- if not model_weights_serialized or not model_shapes:
- return None
-
- i, j, model_weights = 0, 0, []
- model_weights_serialized = np.fromstring(model_weights_serialized, dtype=np.float32)
-
- total_model_shape = \
- sum([reduce(lambda x, y: x * y, ls) for ls in model_shapes])
- total_weights_shape = model_weights_serialized.size
- _assert(total_model_shape == total_weights_shape,
- "Number of elements in model weights({0}) doesn't match model({1})."\
- .format(total_weights_shape, total_model_shape))
- while j < len(model_shapes):
- next_pointer = i + reduce(lambda x, y: x * y, model_shapes[j])
- weight_arr_portion = model_weights_serialized[i:next_pointer]
- model_weights.append(np.array(weight_arr_portion).reshape(model_shapes[j]))
- i, j = next_pointer, j + 1
- return model_weights
-
-
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
deleted file mode 100644
index 11730cf..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
+++ /dev/null
@@ -1,506 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import plpy
-from keras_model_arch_table import ModelArchSchema
-from model_arch_info import get_num_classes
-from madlib_keras_helper import CLASS_VALUES_COLNAME
-from madlib_keras_helper import COMPILE_PARAMS_COLNAME
-from madlib_keras_helper import DEPENDENT_VARNAME_COLNAME
-from madlib_keras_helper import DEPENDENT_VARTYPE_COLNAME
-from madlib_keras_helper import MODEL_ID_COLNAME
-from madlib_keras_helper import MODEL_ARCH_TABLE_COLNAME
-from madlib_keras_helper import MODEL_WEIGHTS_COLNAME
-from madlib_keras_helper import NORMALIZING_CONST_COLNAME
-from madlib_keras_helper import DISTRIBUTION_KEY_COLNAME
-from madlib_keras_helper import METRIC_TYPE_COLNAME
-from madlib_keras_helper import INTERNAL_GPU_CONFIG
-from madlib_keras_helper import query_model_configs
-
-from utilities.minibatch_validation import validate_bytea_var_for_minibatch
-from utilities.utilities import _assert
-from utilities.utilities import add_postfix
-from utilities.utilities import is_platform_pg
-from utilities.utilities import is_var_valid
-from utilities.validate_args import cols_in_tbl_valid
-from utilities.validate_args import columns_exist_in_table
-from utilities.validate_args import get_expr_type
-from utilities.validate_args import input_tbl_valid
-from utilities.validate_args import output_tbl_valid
-from madlib_keras_wrapper import parse_and_validate_fit_params
-from madlib_keras_wrapper import parse_and_validate_compile_params
-
-class InputValidator:
- @staticmethod
- def validate_predict_evaluate_tables(
- module_name, model_table, model_summary_table, test_table, output_table,
- independent_varname):
- InputValidator._validate_model_weights_tbl(module_name, model_table)
- InputValidator._validate_model_summary_tbl(
- module_name, model_summary_table)
- InputValidator._validate_test_tbl(
- module_name, test_table, independent_varname)
- output_tbl_valid(output_table, module_name)
-
- @staticmethod
- def validate_id_in_test_tbl(module_name, test_table, id_col):
- _assert(is_var_valid(test_table, id_col),
- "{module_name} error: invalid id column "
- "('{id_col}') for test table ({table}).".format(
- module_name=module_name,
- id_col=id_col,
- table=test_table))
-
- @staticmethod
- def validate_predict_byom_tables(module_name, model_arch_table, model_id,
- test_table, id_col, output_table,
- independent_varname):
- InputValidator.validate_model_arch_table(
- module_name, model_arch_table, model_id)
- InputValidator._validate_test_tbl(
- module_name, test_table, independent_varname)
- InputValidator.validate_id_in_test_tbl(module_name, test_table, id_col)
-
- output_tbl_valid(output_table, module_name)
-
-
- @staticmethod
- def validate_pred_type(module_name, pred_type, class_values):
- if not pred_type in ['prob', 'response']:
- plpy.error("{0}: Invalid value for pred_type param ({1}). Must be "\
- "either response or prob.".format(module_name, pred_type))
-
- @staticmethod
- def validate_input_shape(table, independent_varname, input_shape, offset,
- is_minibatched=False):
- """
- Validate if the input shape specified in model architecture is the same
- as the shape of the image specified in the indepedent var of the input
- table.
- offset: This offset is the index of the start of the image array. We also
- need to consider that sql array indexes start from 1
- For ex if the image is of shape [32,32,3] and is minibatched, the image will
- look like [10, 32, 32, 3]. The offset in this case is 1 (start the index at 1) +
- 1 (ignore the buffer size 10) = 2.
- If the image is not batched then it will look like [32, 32 ,3] and the offset in
- this case is 1 (start the index at 1).
- """
-
- if is_minibatched:
- ind_shape_col = add_postfix(independent_varname, "_shape")
- query = """
- SELECT {ind_shape_col} AS shape
- FROM {table}
- LIMIT 1
- """.format(**locals())
- # This query will fail if an image in independent var does not have the
- # same number of dimensions as the input_shape.
- result = plpy.execute(query)[0]['shape']
- result = result[1:]
- else:
- array_upper_query = ", ".join("array_upper({0}, {1}) AS n_{2}".format(
- independent_varname, i+offset, i) for i in range(len(input_shape)))
- query = """
- SELECT {0}
- FROM {1}
- LIMIT 1
- """.format(array_upper_query, table)
-
- # This query will fail if an image in independent var does not have the
- # same number of dimensions as the input_shape.
- result = plpy.execute(query)[0]
-
- _assert(len(result) == len(input_shape),
- "model_keras error: The number of dimensions ({0}) of each image"
- " in model architecture and {1} in {2} ({3}) do not match.".format(
- len(input_shape), independent_varname, table, len(result)))
-
- for i in range(len(input_shape)):
- if is_minibatched:
- key_name = i
- input_shape_from_table = [result[j]
- for j in range(len(input_shape))]
- else:
- key_format = "n_{0}"
- key_name = key_format.format(i)
- input_shape_from_table = [result[key_format.format(j)]
- for j in range(len(input_shape))]
-
- if result[key_name] != input_shape[i]:
- # Construct the shape in independent varname to display
- # meaningful error msg.
- plpy.error("model_keras error: Input shape {0} in the model"
- " architecture does not match the input shape {1} of column"
- " {2} in table {3}.".format(
- input_shape, input_shape_from_table,
- independent_varname, table))
-
- @staticmethod
- def validate_model_arch_table(module_name, model_arch_table, model_id):
- input_tbl_valid(model_arch_table, module_name)
- _assert(model_id is not None,
- "{0}: Invalid model architecture ID.".format(module_name))
-
-
- @staticmethod
- def validate_normalizing_const(module_name, normalizing_const):
- _assert(normalizing_const > 0,
- "{0} error: Normalizing constant has to be greater than 0.".
- format(module_name))
-
- @staticmethod
- def validate_class_values(module_name, class_values, pred_type, model_arch):
- if not class_values:
- return
- num_classes = len(class_values)
- _assert(num_classes == get_num_classes(model_arch),
- "{0}: The number of class values do not match the " \
- "provided architecture.".format(module_name))
- if pred_type == 'prob' and num_classes+1 >= 1600:
- plpy.error({"{0}: The output will have {1} columns, exceeding the "\
- " max number of columns that can be created (1600)".format(
- module_name, num_classes+1)})
-
- @staticmethod
- def validate_model_weights(module_name, model_arch, model_weights):
- _assert(model_weights and model_arch,
- "{0}: Model weights and architecture must be valid.".format(
- module_name))
-
- @staticmethod
- def _validate_model_weights_tbl(module_name, model_table):
- _assert(is_var_valid(model_table, MODEL_WEIGHTS_COLNAME),
- "{module_name} error: column '{model_weights}' "
- "does not exist in model table '{table}'.".format(
- module_name=module_name,
- model_weights=MODEL_WEIGHTS_COLNAME,
- table=model_table))
- _assert(is_var_valid(model_table, ModelArchSchema.MODEL_ARCH),
- "{module_name} error: column '{model_arch}' "
- "does not exist in model table '{table}'.".format(
- module_name=module_name,
- model_arch=ModelArchSchema.MODEL_ARCH,
- table=model_table))
-
- @staticmethod
- def _validate_test_tbl(module_name, test_table, independent_varname):
- input_tbl_valid(test_table, module_name)
- _assert(is_var_valid(test_table, independent_varname),
- "{module_name} error: invalid independent_varname "
- "('{independent_varname}') for test table "
- "({table}).".format(
- module_name=module_name,
- independent_varname=independent_varname,
- table=test_table))
-
- @staticmethod
- def _validate_model_summary_tbl(module_name, model_summary_table):
- input_tbl_valid(model_summary_table, module_name)
- cols_to_check_for = [CLASS_VALUES_COLNAME,
- DEPENDENT_VARNAME_COLNAME,
- DEPENDENT_VARTYPE_COLNAME,
- MODEL_ID_COLNAME,
- MODEL_ARCH_TABLE_COLNAME,
- NORMALIZING_CONST_COLNAME,
- COMPILE_PARAMS_COLNAME,
- METRIC_TYPE_COLNAME]
- _assert(columns_exist_in_table(
- model_summary_table, cols_to_check_for),
- "{0} error: One or more expected columns missing in model "
- "summary table ('{1}'). The expected columns are {2}.".format(
- module_name, model_summary_table, cols_to_check_for))
-
- @staticmethod
- def _validate_gpu_config(module_name, source_table, accessible_gpus_for_seg):
-
- summary_table = add_postfix(source_table, "_summary")
- gpu_config = plpy.execute(
- "SELECT {0} FROM {1}".format(INTERNAL_GPU_CONFIG, summary_table)
- )[0][INTERNAL_GPU_CONFIG]
- if gpu_config == 'all_segments':
- _assert(0 not in accessible_gpus_for_seg,
- "{0} error: Host(s) are missing gpus.".format(module_name))
- else:
- for i in gpu_config:
- _assert(accessible_gpus_for_seg[i] != 0,
- "{0} error: Segment {1} does not have gpu".format(module_name, i))
-
-class FitCommonValidator(object):
- def __init__(self, source_table, validation_table, output_model_table,
- model_arch_table, model_id, dependent_varname,
- independent_varname, num_iterations,
- metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg, module_name):
- self.source_table = source_table
- self.validation_table = validation_table
- self.output_model_table = output_model_table
- self.model_arch_table = model_arch_table
- self.model_id = model_id
- self.dependent_varname = dependent_varname
- self.independent_varname = independent_varname
- self.dep_shape_col = add_postfix(dependent_varname, "_shape")
- self.ind_shape_col = add_postfix(independent_varname, "_shape")
- self.metrics_compute_frequency = metrics_compute_frequency
- self.warm_start = warm_start
- self.num_iterations = num_iterations
- self.source_summary_table = None
- if self.source_table:
- self.source_summary_table = add_postfix(
- self.source_table, "_summary")
- if self.output_model_table:
- self.output_summary_model_table = add_postfix(
- self.output_model_table, "_summary")
- self.accessible_gpus_for_seg = accessible_gpus_for_seg
- self.module_name = module_name
- self._validate_common_args()
- if use_gpus:
- InputValidator._validate_gpu_config(self.module_name,
- self.source_table, self.accessible_gpus_for_seg)
-
- def _validate_common_args(self):
- _assert(self.num_iterations > 0,
- "{0}: Number of iterations cannot be < 1.".format(self.module_name))
- _assert(self._is_valid_metrics_compute_frequency(),
- "{0}: metrics_compute_frequency must be in the range (1 - {1}).".format(
- self.module_name, self.num_iterations))
- input_tbl_valid(self.source_table, self.module_name)
- input_tbl_valid(self.source_summary_table, self.module_name,
- error_suffix_str="Please ensure that the source table ({0}) "
- "has been preprocessed by "
- "the image preprocessor.".format(self.source_table))
- cols_in_tbl_valid(self.source_summary_table, [CLASS_VALUES_COLNAME,
- NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME,
- 'dependent_varname', 'independent_varname'], self.module_name)
- if not is_platform_pg():
- cols_in_tbl_valid(self.source_table, [DISTRIBUTION_KEY_COLNAME], self.module_name)
-
- # Source table and validation tables must have the same schema
- self._validate_input_table(self.source_table)
- validate_bytea_var_for_minibatch(self.source_table,
- self.dependent_varname)
-
- self._validate_validation_table()
- if self.warm_start:
- input_tbl_valid(self.output_model_table, self.module_name)
- input_tbl_valid(self.output_summary_model_table, self.module_name)
- else:
- output_tbl_valid(self.output_model_table, self.module_name)
- output_tbl_valid(self.output_summary_model_table, self.module_name)
-
- def _validate_input_table(self, table):
- _assert(is_var_valid(table, self.independent_varname),
- "{module_name}: invalid independent_varname "
- "('{independent_varname}') for table ({table}). "
- "Please ensure that the input table ({table}) "
- "has been preprocessed by the image preprocessor.".format(
- module_name=self.module_name,
- independent_varname=self.independent_varname,
- table=table))
-
- _assert(is_var_valid(table, self.dependent_varname),
- "{module_name}: invalid dependent_varname "
- "('{dependent_varname}') for table ({table}). "
- "Please ensure that the input table ({table}) "
- "has been preprocessed by the image preprocessor.".format(
- module_name=self.module_name,
- dependent_varname=self.dependent_varname,
- table=table))
-
- _assert(is_var_valid(table, self.ind_shape_col),
- "{module_name}: invalid independent_var_shape "
- "('{ind_shape_col}') for table ({table}). "
- "Please ensure that the input table ({table}) "
- "has been preprocessed by the image preprocessor.".format(
- module_name=self.module_name,
- ind_shape_col=self.ind_shape_col,
- table=table))
-
- _assert(is_var_valid(table, self.dep_shape_col),
- "{module_name}: invalid dependent_var_shape "
- "('{dep_shape_col}') for table ({table}). "
- "Please ensure that the input table ({table}) "
- "has been preprocessed by the image preprocessor.".format(
- module_name=self.module_name,
- dep_shape_col=self.dep_shape_col,
- table=table))
-
- if not is_platform_pg():
- _assert(is_var_valid(table, DISTRIBUTION_KEY_COLNAME),
- "{module_name}: missing distribution key "
- "('{dist_key_col}') for table ({table}). "
- "Please ensure that the input table ({table}) "
- "has been preprocessed by the image preprocessor.".format(
- module_name=self.module_name,
- dist_key_col=DISTRIBUTION_KEY_COLNAME,
- table=table))
-
- def _is_valid_metrics_compute_frequency(self):
- return self.metrics_compute_frequency is None or \
- (self.metrics_compute_frequency >= 1 and \
- self.metrics_compute_frequency <= self.num_iterations)
-
- def _validate_validation_table(self):
- if self.validation_table and self.validation_table.strip() != '':
- input_tbl_valid(self.validation_table, self.module_name)
- self._validate_input_table(self.validation_table)
- dependent_vartype = get_expr_type(self.dependent_varname,
- self.validation_table)
- _assert(dependent_vartype == 'bytea',
- "Dependent variable column {0} in validation table {1} should be "
- "a bytea and also one hot encoded.".format(
- self.dependent_varname, self.validation_table))
-
-
- def validate_input_shapes(self, input_shape):
- InputValidator.validate_input_shape(self.source_table, self.independent_varname,
- input_shape, 2, True)
- if self.validation_table:
- InputValidator.validate_input_shape(
- self.validation_table, self.independent_varname,
- input_shape, 2, True)
-
-
-class FitInputValidator(FitCommonValidator):
- def __init__(self, source_table, validation_table, output_model_table,
- model_arch_table, model_id, dependent_varname,
- independent_varname, num_iterations,
- metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg):
-
- self.module_name = 'madlib_keras_fit'
- super(FitInputValidator, self).__init__(source_table,
- validation_table,
- output_model_table,
- model_arch_table,
- model_id,
- dependent_varname,
- independent_varname,
- num_iterations,
- metrics_compute_frequency,
- warm_start,
- use_gpus,
- accessible_gpus_for_seg,
- self.module_name)
- InputValidator.validate_model_arch_table(self.module_name, self.model_arch_table,
- self.model_id)
-
-class FitMultipleInputValidator(FitCommonValidator):
- def __init__(self, source_table, validation_table, output_model_table,
- model_selection_table, model_selection_summary_table, dependent_varname,
- independent_varname, num_iterations, model_info_table, mst_key_col,
- model_arch_table_col, metrics_compute_frequency, warm_start,
- use_gpus, accessible_gpus_for_seg):
-
- self.module_name = 'madlib_keras_fit_multiple'
-
- input_tbl_valid(model_selection_table, self.module_name)
- input_tbl_valid(model_selection_summary_table, self.module_name,
- error_suffix_str="Please ensure that the model selection table ({0}) "
- "has been created by "
- "load_model_selection_table().".format(
- model_selection_table))
- self.msts, self.model_arch_table = query_model_configs(
- model_selection_table, model_selection_summary_table,
- mst_key_col, model_arch_table_col)
- if warm_start:
- input_tbl_valid(model_info_table, self.module_name)
- else:
- output_tbl_valid(model_info_table, self.module_name)
- super(FitMultipleInputValidator, self).__init__(source_table,
- validation_table,
- output_model_table,
- self.model_arch_table,
- None,
- dependent_varname,
- independent_varname,
- num_iterations,
- metrics_compute_frequency,
- warm_start,
- use_gpus,
- accessible_gpus_for_seg,
- self.module_name)
-
-class MstLoaderInputValidator():
- def __init__(self,
- model_arch_table,
- model_selection_table,
- model_selection_summary_table,
- model_id_list,
- compile_params_list,
- fit_params_list
- ):
- self.model_arch_table = model_arch_table
- self.model_selection_table = model_selection_table
- self.model_selection_summary_table = model_selection_summary_table
- self.model_id_list = model_id_list
- self.compile_params_list = compile_params_list
- self.fit_params_list = fit_params_list
- self.module_name = 'load_model_selection_table'
- self._validate_input_args()
-
- def _validate_input_args(self):
- self._validate_input_output_tables()
- self._validate_model_ids()
- self._validate_compile_and_fit_params()
-
- def _validate_model_ids(self):
- model_id_str = '({0})'\
- .format(','.join([str(x) for x in self.model_id_list]))
- query = """
- SELECT count(model_id)
- FROM {self.model_arch_table}
- WHERE model_id IN {model_id_str}
- """.format(**locals())
- res = int(plpy.execute(query)[0]['count'])
- _assert(
- res == len(self.model_id_list),
- "{0}: One or more model_id of {1} not found in table {2}".format(
- self.module_name,
- model_id_str,
- self.model_arch_table
- )
- )
- def _validate_compile_and_fit_params(self):
- if not self.fit_params_list:
- plpy.error("fit_params_list cannot be NULL")
- for fit_params in self.fit_params_list:
- try:
- res = parse_and_validate_fit_params(fit_params)
- except Exception as e:
- plpy.error(
- """Fit param check failed for: {0} \n
- {1}
- """.format(fit_params, str(e)))
- if not self.compile_params_list:
- plpy.error( "compile_params_list cannot be NULL")
- for compile_params in self.compile_params_list:
- try:
- res = parse_and_validate_compile_params(compile_params)
- except Exception as e:
- plpy.error(
- """Compile param check failed for: {0} \n
- {1}
- """.format(compile_params, str(e)))
-
- def _validate_input_output_tables(self):
- input_tbl_valid(self.model_arch_table, self.module_name)
- output_tbl_valid(self.model_selection_table, self.module_name)
- output_tbl_valid(self.model_selection_summary_table, self.module_name)
-
-
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_wrapper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_wrapper.py_in
deleted file mode 100644
index 541d370..0000000
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_wrapper.py_in
+++ /dev/null
@@ -1,333 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import ast
-import os
-import plpy
-from math import ceil
-
-from keras import backend as K
-from keras import utils as keras_utils
-from keras.optimizers import *
-
-import keras.optimizers as opt
-import keras.losses as losses
-
-import madlib_keras_serializer
-import madlib_keras_gpu_info
-from utilities.utilities import _assert
-from utilities.utilities import is_platform_pg
-
-CUDA_VISIBLE_DEVICES_KEY = 'CUDA_VISIBLE_DEVICES'
-#######################################################################
-########### Keras specific functions #####
-#######################################################################
-
-def set_cuda_env(value):
- """
- :param value: -1 to disable gpu
- :return:
- """
- os.environ[CUDA_VISIBLE_DEVICES_KEY] = value
-
-def reset_cuda_env(value):
- """
- This function will reset the cuda env variable. This should only be called
- if set_cuda_env was called previously.
- :param value:
- """
- if value:
- set_cuda_env(value)
- else:
- if CUDA_VISIBLE_DEVICES_KEY in os.environ:
- del os.environ[CUDA_VISIBLE_DEVICES_KEY]
-
-def get_device_name_and_set_cuda_env(gpu_count, seg):
-
- if gpu_count > 0:
- device_name = '/gpu:0'
- if is_platform_pg():
- cuda_visible_dev = ','.join([str(i) for i in range(gpu_count)])
- else:
- cuda_visible_dev = str(seg % gpu_count)
- set_cuda_env(cuda_visible_dev)
- else: # cpu only
- device_name = '/cpu:0'
- set_cuda_env('-1')
- return device_name
-
-def set_keras_session(device_name, gpu_count, segments_per_host):
- with K.tf.device(device_name):
- session = get_keras_session(device_name, gpu_count, segments_per_host)
- K.set_session(session)
-
-def get_keras_session(device_name, gpu_count, segments_per_host):
- config = K.tf.ConfigProto()
- if gpu_count > 0:
- memory_fraction = get_gpu_memory_fraction(gpu_count, segments_per_host)
- config.gpu_options.allow_growth = False
- config.gpu_options.per_process_gpu_memory_fraction = memory_fraction
- session = tf.Session(config=config)
- return session
-
-def clear_keras_session(sess = None):
- if sess is None:
- sess = K.get_session()
- K.clear_session()
- sess.close()
-
-
-def get_gpu_memory_fraction(gpu_count, segments_per_host):
- """
- We cap the gpu memory usage to 90% of the total available gpu memory.
- This 90% is evenly distributed among the segments per gpu.
- :param gpu_count:
- :param segments_per_host:
- :return:
- """
- return 0.9 / ceil(1.0 * segments_per_host / gpu_count)
-
-def get_model_shapes(model):
- model_shapes = []
- for a in model.get_weights():
- model_shapes.append(a.shape)
- return model_shapes
-
-def compile_and_set_weights(segment_model, compile_params, device_name,
- serialized_weights):
- model_shapes = get_model_shapes(segment_model)
- compile_model(segment_model, compile_params)
- model_weights = madlib_keras_serializer.deserialize_as_nd_weights(
- serialized_weights, model_shapes)
- segment_model.set_weights(model_weights)
-
-# TODO: This can be refactored to be part of compile_and_set_weights(),
-# by making compile_params an optional param in that function. Doing that
-# now might create more merge conflicts with other JIRAs, so get to this later.
-def set_model_weights(segment_model, serialized_weights):
- model_shapes = get_model_shapes(segment_model)
- model_weights = madlib_keras_serializer.deserialize_as_nd_weights(
- serialized_weights, model_shapes)
- segment_model.set_weights(model_weights)
-
-"""
-Used to convert compile_params and fit_params to actual argument dictionaries
-"""
-
-def convert_string_of_args_to_dict(str_of_args):
- """Uses parenthases matching algorithm to intelligently convert
- a string with valid python code into an argument dictionary"""
- stack = []
- dual = {
- '(' : ')',
- '[' : ']',
- '{' : '}',
- }
- result_str = ""
- key_str = ""
- compile_dict = {}
- for char in str_of_args:
- if char in dual.keys():
- stack.append(char)
- result_str += char
- elif char in dual.values() and stack:
- if dual[stack[-1]] == char:
- stack.pop(-1)
- result_str += char
- elif not stack and char == "=":
- key_str = result_str
- result_str = ""
- elif not stack and char == ",":
- value_str = result_str
- result_str = ""
- compile_dict[key_str.strip()]=value_str.strip().strip('\'')
- else:
- result_str += char
- value_str = result_str
- compile_dict[key_str.strip()]=value_str.strip().strip('\'')
- return compile_dict
-
-def get_metrics_from_compile_param(str_of_args):
- compile_dict = convert_string_of_args_to_dict(str_of_args)
- metrics = None
- ckey = 'metrics'
- if ckey in compile_dict:
- try:
- metrics = ast.literal_eval(compile_dict[ckey])
- except ValueError:
- plpy.error(("Invalid input value for parameter {0}, "
- "please refer to the documentation").format(ckey))
- return metrics
-
-# Parse the compile parameters and the optimizer.
-def parse_and_validate_compile_params(str_of_args):
- """
- Args:
- @param: str_of_args The string of arguments given by the user
- Returns:
- opt_name: Name of the optimizer
- opt_args: Arguments for the optimizer
- compile_dict: Dictionary of arguments for keras.compile
- """
- literal_eval_compile_params = ['metrics', 'loss_weights',
- 'weighted_metrics', 'sample_weight_mode']
- accepted_compile_params = literal_eval_compile_params + ['optimizer', 'loss']
-
- compile_dict = convert_string_of_args_to_dict(str_of_args)
- compile_dict = validate_and_literal_eval_keys(compile_dict,
- literal_eval_compile_params,
- accepted_compile_params)
- _assert('optimizer' in compile_dict, "optimizer is a required parameter for compile")
- opt_name, opt_args = parse_optimizer(compile_dict)
-
- _assert('loss' in compile_dict, "loss is a required parameter for compile")
- validate_compile_param_types(compile_dict)
- _validate_metrics(compile_dict)
- return (opt_name, opt_args, compile_dict)
-
-def _validate_metrics(compile_dict):
- _assert('metrics' not in compile_dict.keys() or
- compile_dict['metrics'] is None or
- type(compile_dict['metrics']) is list,
- "wrong input type for compile parameter metrics: multi-output model"
- "and user defined metrics are not supported yet, please pass a list")
- if 'metrics' in compile_dict and compile_dict['metrics']:
- unsupported_metrics_list = ['sparse_categorical_accuracy',
- 'sparse_categorical_crossentropy', 'top_k_categorical_accuracy',
- 'sparse_top_k_categorical_accuracy']
- _assert(len(compile_dict['metrics']) == 1,
- "Only one metric at a time is supported.")
- _assert(compile_dict['metrics'][0] not in unsupported_metrics_list,
- "Metric {0} is not supported.".format(compile_dict['metrics'][0]))
-
-# Parse the optimizer name and params.
-def parse_optimizer(compile_dict):
- """
- Args:
- @param: compile_dict Dictionary of arguments for keras.compile
- Returns:
- opt_name: Name of the optimizer
- opt_args: Arguments for the optimizer
- """
- opt_split = compile_dict['optimizer'].split('(')
- opt_name = opt_split[0]
- optimizers = get_optimizers()
- _assert(opt_name in optimizers,
- "model_keras error: invalid optimizer name: {0}".format(opt_name))
-
- # If we use only the optimizer name
- if len(opt_split) == 1:
- final_args = None
- # If we use optimizer object with no params
- elif opt_split[1] == ')':
- final_args = None
- # If we give parameters to the optimizer
- else:
- opt_params = opt_split[1][:-1]
- opt_params_array = opt_params.split(',')
- opt_params_clean = map(split_and_strip, opt_params_array)
- key_value_params = { x[0] : x[1] for x in opt_params_clean}
-
- final_args = {}
- for key,value in key_value_params.iteritems():
- if value == 'None':
- final_args[key] = None
- elif value == 'True' or value == 'False':
- final_args[key] = bool(value)
- else:
- final_args[key] = float(value)
- return (opt_name,final_args)
-
-
-# Parse the fit parameters into a dictionary.
-def parse_and_validate_fit_params(fit_param_str):
-
- if fit_param_str:
- fit_params_dict = convert_string_of_args_to_dict(fit_param_str)
-
- literal_eval_fit_params = ['batch_size','epochs','verbose',
- 'class_weight','initial_epoch','steps_per_epoch']
- accepted_fit_params = literal_eval_fit_params + ['shuffle']
-
- fit_params_dict = validate_and_literal_eval_keys(fit_params_dict,
- literal_eval_fit_params,
- accepted_fit_params)
- if 'shuffle' in fit_params_dict:
- shuffle_value = fit_params_dict['shuffle']
- if shuffle_value == 'True' or shuffle_value == 'False':
- fit_params_dict['shuffle'] = bool(shuffle_value)
-
- return fit_params_dict
- else:
- return {}
-
-# Validate the keys of the given dictionary and run literal_eval on the
-# user-defined subset
-def validate_and_literal_eval_keys(keys_dict, literal_eval_list, accepted_list):
- for ckey in keys_dict.keys():
- _assert(ckey in accepted_list,
- "{0} is not currently accepted as a parameter. ".format(ckey))
- if ckey in literal_eval_list:
- try:
- keys_dict[ckey] = ast.literal_eval(keys_dict[ckey])
- except ValueError:
- plpy.error(("invalid input value for parameter {0}, "
- "please refer to the documentation").format(ckey))
- return keys_dict
-
-# Split and strip the whitespace of key=value formatted strings
-def split_and_strip(x):
- y = x.split('=')
- return (y[0].strip(),y[1].strip())
-
-# Return the list of keras optimizers
-def get_optimizers():
- optimizers = dict()
- names = dir(opt)
- for n in names:
- optimizer = eval('opt.' + n)
- if optimizer.__class__ == type and optimizer.__base__ == opt.Optimizer:
- optimizers[n] = optimizer
- return optimizers
-
-# Run the keras.compile with the given parameters
-def compile_model(model, compile_params):
- optimizers = get_optimizers()
- (opt_name,final_args,compile_dict) = parse_and_validate_compile_params(compile_params)
- compile_dict['optimizer'] = optimizers[opt_name](**final_args) if final_args else opt_name
- model.compile(**compile_dict)
-
-def validate_compile_param_types(compile_dict):
- _assert('loss_weights' not in compile_dict.keys() or
- compile_dict['loss_weights'] is None or
- type(compile_dict['loss_weights']) is list or
- type(compile_dict['loss_weights']) is dict,
- "wrong input type for compile parameter loss_weights: only list "
- "and dictionary are supported.")
-
- _assert('weighted_metrics' not in compile_dict.keys() or
- compile_dict['weighted_metrics'] is None or
- type(compile_dict['weighted_metrics']) is list,
- "wrong input type for compile parameter weighted_metrics: only list "
- "is supported.")
-
- _assert('sample_weight_mode' not in compile_dict.keys() or
- compile_dict['sample_weight_mode'] is None or
- compile_dict['sample_weight_mode'] == "temporal",
- """compile parameter sample_weight_mode can only be "temporal" or None""")
diff --git a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in b/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
deleted file mode 100644
index 3a225cc..0000000
--- a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
+++ /dev/null
@@ -1,104 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-m4_changequote(`<!', `!>')
-
-import sys
-import json
-import plpy
-from keras_model_arch_table import ModelArchSchema
-
-def _get_layers(model_arch):
- d = json.loads(model_arch)
- config = d['config']
- if type(config) == list:
- return config # In keras 2.1.x, all models are sequential
- elif type(config) == dict and 'layers' in config:
- layers = config['layers']
- if type(layers) == list:
- return config['layers'] # In keras 2.x, only sequential models are supported
- plpy.error("Unable to read model architecture JSON.")
-
-def get_input_shape(model_arch):
- arch_layers = _get_layers(model_arch)
- if 'batch_input_shape' in arch_layers[0]['config']:
- return arch_layers[0]['config']['batch_input_shape'][1:]
- plpy.error('Unable to get input shape from model architecture.')
-
-def get_num_classes(model_arch):
- """
- We assume that the last dense layer in the model architecture contains the num_classes (units)
- An example can be:
- ```
- ...
- model.add(Flatten())
- model.add(Dense(512))
- model.add(Activation('relu'))
- model.add(Dropout(0.5))
- model.add(Dense(num_classes))
- model.add(Activation('softmax'))
- ```
- where activation can be after the dense layer.
- :param model_arch:
- :return:
- """
- arch_layers = _get_layers(model_arch)
- i = len(arch_layers) - 1
- while i >= 0:
- if 'units' in arch_layers[i]['config']:
- return arch_layers[i]['config']['units']
- i -= 1
- plpy.error('Unable to get number of classes from model architecture.')
-
-def get_model_arch_layers_str(model_arch):
- arch_layers = _get_layers(model_arch)
- layers = "Model arch layers:\n"
- first = True
- for layer in arch_layers:
- if first:
- first = False
- else:
- layers += " |\n"
- layers += " V\n"
- class_name = layer['class_name']
- config = layer['config']
- if class_name == 'Dense':
- layers += "{1}[{2}]\n".format(class_name, config['units'])
- else:
- layers += "{1}\n".format(class_name)
- return layers
-
-def get_model_arch_weights(model_arch_table, model_id):
-
- #assume validation is already called
- model_arch_query = "SELECT {0}, {1} FROM {2} WHERE {3} = {4}".format(
- ModelArchSchema.MODEL_ARCH, ModelArchSchema.MODEL_WEIGHTS,
- model_arch_table, ModelArchSchema.MODEL_ID,
- model_id)
- model_arch_result = plpy.execute(model_arch_query)
- if not model_arch_result:
- plpy.error("no model arch found in table {0} with id {1}".format(
- model_arch_table, model_id))
-
- model_arch_result = model_arch_result[0]
-
- model_arch = model_arch_result[ModelArchSchema.MODEL_ARCH]
- model_weights = model_arch_result[ModelArchSchema.MODEL_WEIGHTS]
-
- return model_arch, model_weights
diff --git a/src/ports/postgres/modules/deep_learning/predict_input_params.py_in b/src/ports/postgres/modules/deep_learning/predict_input_params.py_in
deleted file mode 100644
index fdab1e4..0000000
--- a/src/ports/postgres/modules/deep_learning/predict_input_params.py_in
+++ /dev/null
@@ -1,67 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import plpy
-from keras_model_arch_table import ModelArchSchema
-from utilities.utilities import add_postfix
-from utilities.validate_args import input_tbl_valid
-
-from madlib_keras_helper import CLASS_VALUES_COLNAME
-from madlib_keras_helper import DEPENDENT_VARNAME_COLNAME
-from madlib_keras_helper import DEPENDENT_VARTYPE_COLNAME
-from madlib_keras_helper import MODEL_WEIGHTS_COLNAME
-from madlib_keras_helper import NORMALIZING_CONST_COLNAME
-from madlib_keras_helper import create_summary_view
-
-class PredictParamsProcessor:
- def __init__(self, model_table, module_name, mst_key):
- self.module_name = module_name
- self.model_table = model_table
- self.mst_key = mst_key
- self.model_summary_table = add_postfix(self.model_table, '_summary')
- self.mult_where_clause = ""
- if self.mst_key:
- self.model_summary_table = create_summary_view(self.module_name,
- self.model_table,
- self.mst_key)
- self.mult_where_clause = "WHERE mst_key = {0}".format(self.mst_key)
- input_tbl_valid(self.model_summary_table, self.module_name)
- self.model_summary_dict = self._get_dict_for_table(self.model_summary_table)
- self.model_arch_dict = self._get_dict_for_table(self.model_table)
-
- def _get_dict_for_table(self, table_name):
- return plpy.execute("SELECT * FROM {0} {1}".format(table_name, self.mult_where_clause), 1)[0]
-
- def get_class_values(self):
- return self.model_summary_dict[CLASS_VALUES_COLNAME]
-
- def get_dependent_varname(self):
- return self.model_summary_dict[DEPENDENT_VARNAME_COLNAME]
-
- def get_dependent_vartype(self):
- return self.model_summary_dict[DEPENDENT_VARTYPE_COLNAME]
-
- def get_model_arch(self):
- return self.model_arch_dict[ModelArchSchema.MODEL_ARCH]
-
- def get_model_weights(self):
- return self.model_arch_dict[MODEL_WEIGHTS_COLNAME]
-
- def get_normalizing_const(self):
- return self.model_summary_dict[NORMALIZING_CONST_COLNAME]
diff --git a/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in b/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in
deleted file mode 100644
index 7c6c5c3..0000000
--- a/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in
+++ /dev/null
@@ -1,645 +0,0 @@
-/* ----------------------------------------------------------------------- *//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ----------------------------------------------------------------------- */
-m4_include(`SQLCommon.m4')
-m4_changequote(`<!', `!>')
-
-DROP TABLE IF EXISTS data_preprocessor_input;
-CREATE TABLE data_preprocessor_input(id serial, x double precision[], label TEXT);
-INSERT INTO data_preprocessor_input(x, label) VALUES
-(ARRAY[1,2,3,4,5,6], 'a'),
-(ARRAY[11,2,3,4,5,6], 'a'),
-(ARRAY[11,22,33,4,5,6], 'a'),
-(ARRAY[11,22,33,44,5,6], 'a'),
-(ARRAY[11,22,33,44,65,6], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,144,65,56], 'a'),
-(ARRAY[11,22,233,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'b'),
-(ARRAY[11,22,33,44,65,56], 'b'),
-(ARRAY[11,22,33,44,65,56], 'b');
-
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'id',
- 'x',
- 5);
-
--- Divide two numbers and round up to the nearest integer
-CREATE FUNCTION divide_roundup(numerator NUMERIC, denominator NUMERIC)
-RETURNS INTEGER AS
-$$
- SELECT (ceil($1 / $2)::INTEGER);
-$$ LANGUAGE SQL;
-
--- num_buffers_calc() represents the num_buffers value that should be
--- calculated by the preprocessor.
--- For postgres, just need total rows / buffer_size rounded up.
--- For greenplum, we take that result, and round up to the nearest multiple
--- of num_segments.
-CREATE FUNCTION num_buffers_calc(rows_in_tbl INTEGER, buffer_size INTEGER)
-RETURNS INTEGER AS
-$$
-m4_ifdef(<!__POSTGRESQL__!>,
- <! SELECT divide_roundup($1, $2); !>,
- <! SELECT (COUNT(*)::INTEGER) * divide_roundup(divide_roundup($1, $2), COUNT(*)) FROM gp_segment_configuration
- WHERE role = 'p' AND content != -1; !>
-)
-$$ LANGUAGE SQL;
-
--- num_buffers() represents the actual number of buffers expected to
--- be returned in the output table.
--- For postgres, this should always be the same as num_buffers_calc()
--- (as long as rows_in_tbl > 0, which should be validated elsewhere)
--- For greenplum, this can be less than num_buffers_calc() in
--- the special case where there is only one row per buffer. In
--- that case, the buffers in the output table will be equal to
--- the number of rows in the input table. This can only happen
--- if rows_in_tbl < num_segments and is the only case where the
--- number of buffers on each segment will not be exactly equal
-CREATE FUNCTION num_buffers(rows_in_tbl INTEGER, buffer_size INTEGER)
-RETURNS INTEGER AS
-$$
- SELECT LEAST(num_buffers_calc($1, $2), $1);
-$$ LANGUAGE SQL;
-
-CREATE FUNCTION buffer_size(rows_in_tbl INTEGER, requested_buffer_size INTEGER)
-RETURNS INTEGER AS
-$$
- SELECT divide_roundup($1, num_buffers($1, $2));
-$$ LANGUAGE SQL;
-
-SELECT assert(COUNT(*) = num_buffers(17, 5),
- 'Incorrect number of buffers in data_preprocessor_input_batch.')
-FROM data_preprocessor_input_batch;
-
-SELECT assert(independent_var_shape[2]=6, 'Incorrect image shape ' || independent_var_shape[2])
-FROM data_preprocessor_input_batch WHERE buffer_id=0;
-
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1])
-FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1])
-FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=1;
-
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1])
-FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=2;
-
-SELECT assert(total_images = 17, 'Incorrect total number of images! Last buffer has incorrect size?')
-FROM (SELECT SUM(independent_var_shape[1]) AS total_images FROM data_preprocessor_input_batch) a;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length ' || octet_length(independent_var)::TEXT)
-FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
-
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-SELECT validation_preprocessor_dl(
- 'data_preprocessor_input',
- 'validation_out',
- 'id',
- 'x',
- 'data_preprocessor_input_batch',
- 5);
-
-SELECT assert(COUNT(*) = num_buffers(17, 5),
- 'Incorrect number of buffers in validation_out.')
-FROM validation_out;
-
-SELECT assert(independent_var_shape[2]=6, 'Incorrect image shape.')
-FROM data_preprocessor_input_batch WHERE buffer_id=0;
-
-SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size.')
-FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=1;
-
-SELECT assert(total_images = 17, 'Incorrect total number of images! Last buffer has incorrect size?')
-FROM (SELECT SUM(independent_var_shape[1]) AS total_images FROM data_preprocessor_input_batch) a;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 5) buffer_size) a, validation_out WHERE buffer_id=0;
-
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'label',
- 'x');
-
--- Test data is evenly distributed across all segments (GPDB only)
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'id',
- 'x',
- 1);
-
--- This test expects that total number of images(17 for input table data_preprocessor_input)
--- are equally distributed across all segments.
--- Therefore, after preprocessing seg0 will have 17/(# of segs) buffers.
-SELECT gp_segment_id, assert((SELECT divide_roundup(17, count(*)) from gp_segment_configuration WHERE role = 'p' and content != -1) - COUNT(*) <= 1, 'Even distribution of buffers failed. Seeing ' || count(*) || ' buffers.')
- FROM data_preprocessor_input_batch GROUP BY 1;
-SELECT assert(__internal_gpu_config__ = 'all_segments', 'Missing column in summary table')
- FROM data_preprocessor_input_batch_summary;
-
--- Test validation data is evenly distributed across all segments (GPDB only)
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-SELECT validation_preprocessor_dl(
- 'data_preprocessor_input',
- 'validation_out',
- 'id',
- 'x',
- 'data_preprocessor_input_batch',
- 1);
-SELECT gp_segment_id, assert((SELECT divide_roundup(17, count(*)) from gp_segment_configuration WHERE role = 'p' and content != -1) - COUNT(*) <= 1, 'Even distribution of buffers failed. Seeing ' || count(*) || ' buffers.')
- FROM validation_out GROUP BY 1;
-SELECT assert(__internal_gpu_config__ = 'all_segments', 'Missing column in validation summary table')
-FROM validation_out_summary;
-
--- Test data distributed on specified segments
-DROP TABLE IF EXISTS segments_to_use;
-CREATE TABLE segments_to_use (dbid INTEGER, notes TEXT);
-INSERT INTO segments_to_use VALUES (2, 'GPU segment');
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'id',
- 'x',
- 1,
- NULL,
- NULL,
- 'segments_to_use');
-SELECT assert(count(DISTINCT(gp_segment_id)) = 1, 'Fail to distribute data on segment0')
-FROM data_preprocessor_input_batch;
-SELECT assert(count(*) = 17, 'Fail to distribute all data on segment0')
-FROM data_preprocessor_input_batch;
-SELECT assert(__internal_gpu_config__ = ARRAY[0], 'Invalid column value in summary table')
-FROM data_preprocessor_input_batch_summary;
-
--- Test data distributed on specified segments for validation_preprocessor_dl
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-SELECT validation_preprocessor_dl(
- 'data_preprocessor_input',
- 'validation_out',
- 'id',
- 'x',
- 'data_preprocessor_input_batch',
- 1,
- 'segments_to_use');
-SELECT assert(count(DISTINCT(gp_segment_id)) = 1, 'Failed to distribute validation data on segment0')
-FROM validation_out;
-SELECT assert(count(*) = 17, 'Fail to distribute all validation data on segment0')
-FROM validation_out;
-SELECT assert(__internal_gpu_config__ = ARRAY[0], 'Invalid column value in validation summary table')
-FROM validation_out_summary;
-!>)
-
-DROP TABLE IF EXISTS data_preprocessor_input;
-CREATE TABLE data_preprocessor_input(id serial, x double precision[], y INTEGER, y1 BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]);
-INSERT INTO data_preprocessor_input(x, y, y1, y2, y3, y4, y5) VALUES
-(ARRAY[1,2,3,4,5,6], 4, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,2,3,4,5,6], 3, TRUE, 'c', 4.2, ARRAY[0.0, 1.0], ARRAY[1,0]),
-(ARRAY[11,22,33,4,5,6], 8, TRUE, 'a', 4.0, ARRAY[0.0, 1.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,5,6], 2, FALSE, 'a', 4.2, ARRAY[0.0, 1.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,6], 5, TRUE, 'b', 4.0, ARRAY[0.0, 1.0], ARRAY[0,1]),
-(ARRAY[11,22,33,44,65,56], 6, TRUE, 'a', 5.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], 2, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], 10, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], 3, TRUE, 'b', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], 7, FALSE, 'a', 5.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], 6, TRUE, 'a', 4.0, ARRAY[0.0, 1.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], -6, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,144,65,56], 9, TRUE, 'c', 4.0, ARRAY[0.0, 1.0], ARRAY[1,0]),
-(ARRAY[11,22,233,44,65,56], 0, TRUE, 'a', 5.0, ARRAY[1.0, 0.0], ARRAY[0,1]),
-(ARRAY[11,22,33,44,65,56], 12, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], -3, FALSE, 'a', 4.2, ARRAY[1.0, 0.0], ARRAY[1,0]),
-(ARRAY[11,22,33,44,65,56], -1, TRUE, 'b', 4.0, ARRAY[0.0, 1.0], ARRAY[0,1]);
-
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'y',
- 'x',
- 4,
- 5,
- 16 -- num_classes
- );
-
--- Test that indepdendent vars get divided by 5, by verifying min value goes from 1 to 0.2, and max value from 233 to 46.6
-SELECT assert(relative_error(MIN(x),0.2) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(independent_var)) as x FROM data_preprocessor_input_batch) a;
-SELECT assert(relative_error(MAX(x),46.6) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(independent_var)) as x FROM data_preprocessor_input_batch) a;
--- Test that 1-hot encoded array is of length 16 (num_classes)
-SELECT assert(dependent_var_shape[2] = 16, 'Incorrect one-hot encode dimension with num_classes') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
--- Test summary table
-SELECT assert
- (
- source_table = 'data_preprocessor_input' AND
- output_table = 'data_preprocessor_input_batch' AND
- dependent_varname = 'y' AND
- independent_varname = 'x' AND
- dependent_vartype = 'integer' AND
- class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND
- summary.buffer_size = a.buffer_size AND -- we sort the class values in python
- normalizing_const = 5 AND
- pg_typeof(normalizing_const) = 'real'::regtype AND
- num_classes = 16 AND
- distribution_rules = 'all_segments',
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) FROM (SELECT buffer_size(17, 4) buffer_size) a,
- (SELECT * FROM data_preprocessor_input_batch_summary) summary;
-
---- Test output data type
-SELECT assert(pg_typeof(independent_var) = 'bytea'::regtype, 'Wrong independent_var type') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'Wrong dependent_var type') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-
--- Test for validation data where the input table has only a subset of
--- the classes compared to the original training data
--- (data_preprocessor_input_batch). The one hot encoding must be based
--- on class_values from data_preprocessor_input_batch_summary rather
--- than the class levels found in validation_input.
-DROP TABLE IF EXISTS validation_input;
-CREATE TABLE validation_input(id serial, x_new double precision[], y_new INTEGER, y1 BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]);
-INSERT INTO validation_input(x_new, y_new, y1, y2, y3, y4, y5) VALUES
-(ARRAY[1,2,3,4,5,6], 4, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]);
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-SELECT validation_preprocessor_dl(
- 'validation_input',
- 'validation_out',
- 'y_new',
- 'x_new',
- 'data_preprocessor_input_batch');
--- Hard code 5.0 as the normalizing constant, based on the previous
--- query's input param, to test if normalization is correct.
-SELECT assert(abs(x_new[1]/5.0-(convert_bytea_to_real_array(independent_var))[1]) < 0.0000001, 'Incorrect normalizing in validation table.')
-FROM validation_input, validation_out;
--- Validate if one hot encoding is as expected.
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0}', 'Incorrect one-hot encode dimension with num_classes') FROM
- validation_out WHERE buffer_id = 0;
-
--- Test summary table
-SELECT assert
- (
- source_table = 'validation_input' AND
- output_table = 'validation_out' AND
- dependent_varname = 'y_new' AND
- independent_varname = 'x_new' AND
- dependent_vartype = 'integer' AND
- class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND
- buffer_size = 1 AND -- we sort the class values in python
- normalizing_const = 5 AND
- pg_typeof(normalizing_const) = 'real'::regtype AND
- num_classes = 16,
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) from (select * from validation_out_summary) summary;
-
--- Test one-hot encoding for dependent_var
--- test boolean type
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'y1',
- 'x',
- 4,
- 5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
-SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:2]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
-SELECT assert (dependent_vartype = 'boolean' AND
- class_values = '{f,t}' AND
- num_classes = 2,
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) from (select * from data_preprocessor_input_batch_summary) summary;
-
--- Test to assert the output summary table for validation has the correct
--- num_classes and class_values
-DROP TABLE IF EXISTS validation_input;
-CREATE TABLE validation_input(id serial, x_new double precision[], y INTEGER, y_new BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]);
-INSERT INTO validation_input(x_new, y, y_new, y2, y3, y4, y5) VALUES
-(ARRAY[1,2,3,4,5,6], 4, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]);
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-SELECT validation_preprocessor_dl(
- 'validation_input',
- 'validation_out',
- 'y_new',
- 'x_new',
- 'data_preprocessor_input_batch');
-
-SELECT assert (dependent_vartype = 'boolean' AND
- class_values = '{f,t}' AND
- num_classes = 2,
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) from (select * from validation_out_summary) summary;
--- test text type
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'y2',
- 'x',
- 4,
- 5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
-SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
-SELECT assert (dependent_vartype = 'text' AND
- class_values = '{a,b,c}' AND
- num_classes = 3,
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) from (select * from data_preprocessor_input_batch_summary) summary;
-
--- data_preprocessor_input_batch was created for boolean dependent var,
--- trying to create validation data using this table for reference, but
--- passing a non-boolean dep var should error out.
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-select assert(trap_error($TRAP$SELECT validation_preprocessor_dl(
- 'validation_input',
- 'validation_out',
- 'y_new',
- 'x_new',
- 'data_preprocessor_input_batch');$TRAP$) = 1,
- 'Passing boolean dep var while expecting text dep var should error out.');
-
-
--- test double precision type
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'y3',
- 'x',
- 4,
- 5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0;
-SELECT assert (dependent_vartype = 'double precision' AND
- class_values = '{4.0,4.2,5.0}' AND
- num_classes = 3,
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) from (select * from data_preprocessor_input_batch_summary) summary;
-
--- test double precision array type
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'y4',
- 'x',
- 4,
- 5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
-SELECT assert(relative_error(SUM(y), SUM(y4)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(dependent_var)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y4) as y4 FROM data_preprocessor_input) b;
-SELECT assert (dependent_vartype = 'double precision[]' AND
- class_values IS NULL AND
- num_classes IS NULL,
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) from (select * from data_preprocessor_input_batch_summary) summary;
-
-DROP TABLE IF EXISTS validation_out, validation_out_summary;
-SELECT validation_preprocessor_dl(
- 'validation_input',
- 'validation_out',
- 'y4',
- 'x_new',
- 'data_preprocessor_input_batch');
-
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0}' AND dependent_var_shape[2] = 2, 'Incorrect one-hot encoding for already encoded dep var') FROM
- validation_out WHERE buffer_id = 0;
-
--- test integer array type
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input',
- 'data_preprocessor_input_batch',
- 'y5',
- 'x',
- 4,
- 5);
-SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
-SELECT assert(relative_error(SUM(y), SUM(y5)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(dependent_var)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y5) as y5 FROM data_preprocessor_input) b;
-SELECT assert (dependent_vartype = 'integer[]' AND
- class_values IS NULL AND
- num_classes IS NULL,
- 'Summary Validation failed. Actual:' || __to_char(summary)
- ) from (select * from data_preprocessor_input_batch_summary) summary;
-
--- Test cases with NULL in class values
-DROP TABLE IF EXISTS data_preprocessor_input_null;
-CREATE TABLE data_preprocessor_input_null(id serial, x double precision[], label TEXT);
-INSERT INTO data_preprocessor_input_null(x, label) VALUES
-(ARRAY[1,2,3,4,5,6], 'a'),
-(ARRAY[11,2,3,4,5,6], 'a'),
-(ARRAY[11,22,33,4,5,6], NULL),
-(ARRAY[11,22,33,44,5,6], 'a'),
-(ARRAY[11,22,33,44,65,6], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], NULL),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,44,65,56], NULL),
-(ARRAY[11,22,33,44,65,56], 'a'),
-(ARRAY[11,22,33,144,65,56], 'b'),
-(ARRAY[11,22,233,44,65,56], 'b'),
-(ARRAY[11,22,33,44,65,56], 'b'),
-(ARRAY[11,22,33,44,65,56], 'b'),
-(ARRAY[11,22,33,44,65,56], NULL);
-
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input_null',
- 'data_preprocessor_input_batch',
- 'label',
- 'x',
- 4,
- 5,
- 5 -- num_classes
- );
--- Test summary table if class_values has NULL as a legitimate
--- class label, and also two other NULLs because num_classes=5
--- but table has only 3 distinct class labels (including NULL)
-SELECT assert
- (
- class_values = '{NULL,a,b,NULL,NULL}',
- 'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
- ) from (select * from data_preprocessor_input_batch_summary) summary;
-
-SELECT assert(dependent_var_shape[2] = 5, 'Incorrect one-hot encode dimension') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-
-SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length')
-FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0;
-
--- The same tests, but for validation.
-DROP TABLE IF EXISTS data_preprocessor_input_validation_null;
-CREATE TABLE data_preprocessor_input_validation_null(id serial, x double precision[], label TEXT);
-INSERT INTO data_preprocessor_input_validation_null(x, label) VALUES
-(ARRAY[1,2,3,4,5,6], 'a'),
-(ARRAY[11,2,3,4,5,6], 'b'),
-(ARRAY[11,2,3,4,5,6], 'b'),
-(ARRAY[111,22,33,4,5,6], NULL);
-DROP TABLE IF EXISTS validation_out_batch, validation_out_batch_summary;
-SELECT validation_preprocessor_dl(
- 'data_preprocessor_input_validation_null',
- 'validation_out_batch',
- 'label',
- 'x',
- 'data_preprocessor_input_batch',
- 1
- );
-
--- Test summary table if class_values has NULL as a legitimate
--- class label, and also two other NULLs because num_classes=5
--- but table has only 3 distinct class labels (including NULL)
-SELECT assert
- (
- class_values = '{NULL,a,b,NULL,NULL}',
- 'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
- ) from (select * from validation_out_batch_summary) summary;
-
--- Validate one hot encoding for specific row is correct
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{0,1,0,0,0}' AND dependent_var_shape[2] =5, 'Incorrect normalizing in validation table.')
-FROM data_preprocessor_input_validation_null, validation_out_batch
-WHERE x[1]=1 AND abs((convert_bytea_to_real_array(independent_var))[1] - 0.2::REAL) < 0.00001;
--- Assert one-hot encoding for NULL label
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0,0,0}' AND dependent_var_shape[2] =5, 'Incorrect normalizing in validation table.')
-FROM data_preprocessor_input_validation_null, validation_out_batch
-WHERE x[1]=111 AND abs((convert_bytea_to_real_array(independent_var))[1] - 22.2::REAL) < 0.00001;
-
--- Test the content of 1-hot encoded dep var when NULL is the
--- class label.
-DROP TABLE IF EXISTS data_preprocessor_input_null;
-CREATE TABLE data_preprocessor_input_null(id serial, x double precision[], label TEXT);
-INSERT INTO data_preprocessor_input_null(x, label) VALUES
-(ARRAY[11,22,33,4,5,6], NULL);
-
-DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary;
-SELECT training_preprocessor_dl(
- 'data_preprocessor_input_null',
- 'data_preprocessor_input_batch',
- 'label',
- 'x',
- 4,
- 5,
- 3 -- num_classes
- );
-
--- class_values must be '{NULL,NULL,NULL}' where the first NULL
--- is for the class label seen in data, and the other two NULLs
--- are added as num_classes=3.
-SELECT assert
- (
- class_values = '{NULL,NULL,NULL}',
- 'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
- ) from (select * from data_preprocessor_input_batch_summary) summary;
-
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = 24, 'Incorrect buffer length')
-FROM data_preprocessor_input_batch WHERE buffer_id=0;
--- NULL is treated as a class label, so it should show '1' for the
--- first index
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM
- data_preprocessor_input_batch WHERE buffer_id = 0;
-
--- The same tests for validation.
-DROP TABLE IF EXISTS validation_out_batch, validation_out_batch_summary;
-SELECT validation_preprocessor_dl(
- 'data_preprocessor_input_null',
- 'validation_out_batch',
- 'label',
- 'x',
- 'data_preprocessor_input_batch',
- 4
- );
-
--- class_values must be '{NULL,NULL,NULL}' where the first NULL
--- is for the class label seen in data, and the other two NULLs
--- are added as num_classes=3.
-SELECT assert
- (
- class_values = '{NULL,NULL,NULL}',
- 'Summary Validation failed with NULL data. Actual:' || __to_char(summary)
- ) from (select * from validation_out_batch_summary) summary;
-
-SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM
- validation_out_batch WHERE buffer_id = 0;
-SELECT assert(octet_length(independent_var) = 24, 'Incorrect buffer length')
-FROM data_preprocessor_input_batch WHERE buffer_id=0;
--- NULL is treated as a class label, so it should show '1' for the
--- first index
-SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM
- validation_out_batch WHERE buffer_id = 0;
-
--- Test if validation class values is not a subset of training data class values.
-DROP TABLE IF EXISTS data_preprocessor_input_validation_null;
-CREATE TABLE data_preprocessor_input_validation_null(id serial, x double precision[], label TEXT);
-INSERT INTO data_preprocessor_input_validation_null(x, label) VALUES
-(ARRAY[11,2,3,4,5,6], 'bbbb');
-DROP TABLE IF EXISTS validation_out_batch, validation_out_batch_summary;
-select assert(trap_error($TRAP$SELECT validation_preprocessor_dl(
- 'data_preprocessor_input_validation_null',
- 'validation_out_batch',
- 'label',
- 'x',
- 'data_preprocessor_input_batch',
- 1);$TRAP$) = 1,
- 'validation class values should be a subset of training data class values.');
diff --git a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.ic.sql_in b/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.ic.sql_in
deleted file mode 100644
index 8d82a43..0000000
--- a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.ic.sql_in
+++ /dev/null
@@ -1,31 +0,0 @@
-/* -----------------------------------------------------------------------
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * ----------------------------------------------------------------------- */
-
-/* -----------------------------------------------------------------------------
- * Test Keras Model Arch Table helper functions
- * -------------------------------------------------------------------------- */
-
-SELECT load_keras_model('test239587_keras_model_arch_table','{"a" : 1, "b" : 2, "c" : [4,5,6] }');
-SELECT load_keras_model('test239587_keras_model_arch_table','{"config" : [1,2,3]}');
-SELECT load_keras_model('test239587_keras_model_arch_table','{"config" : [8,4,0]}');
-SELECT delete_keras_model('test239587_keras_model_arch_table',2);
-SELECT delete_keras_model('test239587_keras_model_arch_table',3);
-SELECT delete_keras_model('test239587_keras_model_arch_table',1);
diff --git a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.sql_in b/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.sql_in
deleted file mode 100644
index 1f2009b..0000000
--- a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.sql_in
+++ /dev/null
@@ -1,154 +0,0 @@
-/* -----------------------------------------------------------------------
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * ----------------------------------------------------------------------- */
-
-/* -----------------------------------------------------------------------------
- * Test Keras Model Arch Table helper functions
- * -------------------------------------------------------------------------- */
-
-
-/* Test successful model creation where no table exists */
-DROP TABLE IF EXISTS test_keras_model_arch_table;
-SELECT load_keras_model('test_keras_model_arch_table', '{"a" : 1, "b" : 2, "c" : [4,5,6] }');
-
-SELECT assert(UPPER(atttypid::regtype::TEXT) = 'INTEGER', 'model_id column should be INTEGER type')
- FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass
- AND attname = 'model_id';
-SELECT assert(UPPER(atttypid::regtype::TEXT) = 'JSON', 'model_arch column should be JSON type' ) FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass
- AND attname = 'model_arch';
-SELECT assert(UPPER(atttypid::regtype::TEXT) = 'BYTEA',
- 'model_weights column should be bytea type')
- FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass
- AND attname = 'model_weights';
-
-/* model id should be 1 */
-SELECT assert(model_id = 1, 'Wrong model_id written by load_keras_model')
- FROM test_keras_model_arch_table;
-
-/* model arch should be valid json, with all fields accessible with json operators */
-SELECT assert((model_arch->>'a') = '1', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table;
-SELECT assert((model_arch->>'b') = '2', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table;
-SELECT assert((model_arch->'c')->>0 = '4', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table;
-SELECT assert((model_arch->'c')->>1 = '5', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table;
-SELECT assert((model_arch->'c')->>2 = '6', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table;
-/* model_weights should be set to null, since this is not a warm start */
-SELECT assert(model_weights IS NULL, 'model_weights should be NULL after load_keras_model() called.') FROM test_keras_model_arch_table;
-
-
-/* Test model creation where valid table exists */
-SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}');
-SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [8,4,0]}');
-SELECT assert(model_arch->'config'->>0 = '1', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table WHERE model_id = 2;
-SELECT assert(model_arch->'config'->>1 = '2', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table WHERE model_id = 2;
-SELECT assert(model_arch->'config'->>2 = '3', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table WHERE model_id = 2;
-SELECT assert(model_arch->'config'->>0 = '8', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table WHERE model_id = 3;
-SELECT assert(model_arch->'config'->>1 = '4', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table WHERE model_id = 3;
-SELECT assert(model_arch->'config'->>2 = '0', 'Cannot parse model_arch json in model table.')
- FROM test_keras_model_arch_table WHERE model_id = 3;
-
-/* Test deletion where valid table exists */
-SELECT delete_keras_model('test_keras_model_arch_table', 2);
-SELECT assert(COUNT(model_id) = 0, 'model id 2 should have been deleted!')
- FROM test_keras_model_arch_table WHERE model_id = 2;
-SELECT delete_keras_model('test_keras_model_arch_table', 3);
-SELECT assert(COUNT(model_id) = 0, 'model id 3 should have been deleted!')
- FROM test_keras_model_arch_table WHERE model_id = 3;
- /* Delete a second time, to make sure nothing weird happens.
- * It should archrt to the user that the model_id wasn't found but not
- * raise an exception or change anything. */
-SELECT delete_keras_model('test_keras_model_arch_table', 1);
-SELECT assert(trap_error($$SELECT * from test_keras_model_arch_table$$) = 1,
- 'Table test_keras_model_arch_table should have been deleted.');
-
-SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}');
-DELETE FROM test_keras_model_arch_table;
-
-/* Test deletion where invalid table exists */
-SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}');
-ALTER TABLE test_keras_model_arch_table DROP COLUMN model_id;
-
-/* Test deletion where empty table exists */
-select assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 3)$$) = 1,
- 'Deleting a model in an empty table should generate an exception.');
-
-SELECT assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 1)$$) = 1,
- 'Deleting an invalid table should generate an exception.');
-
-SELECT assert(trap_error($$SELECT load_keras_model('test_keras_model_arch_table', '{"config" : 1}')$$) = 1, 'Passing an invalid table to load_keras_model() should raise exception.');
-
-/* Test deletion where no table exists */
-DROP TABLE IF EXISTS test_keras_model_arch_table;
-SELECT assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 3)$$) = 1,
- 'Deleting a non-existent table should raise exception.');
-
-DROP TABLE IF EXISTS test_keras_model_arch_table;
-SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}', 'dummy weights'::bytea);
-SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}', NULL, 'my name', 'my desc');
-
-/* Test model weights */
-SELECT assert(model_weights = 'dummy weights', 'Incorrect model_weights in the model arch table.')
-FROM test_keras_model_arch_table WHERE model_id = 1;
-SELECT assert(model_weights IS NULL, 'model_weights is not NULL')
-FROM test_keras_model_arch_table WHERE model_id = 2;
-
-/* Test name and description */
-SELECT assert(name IS NULL AND description IS NULL, 'Name or description is not NULL.')
-FROM test_keras_model_arch_table WHERE model_id = 1;
-SELECT assert(name = 'my name' AND description = 'my desc', 'Incorrect name or description in the model arch table.')
-FROM test_keras_model_arch_table WHERE model_id = 2;
-
-
---------------------------- Test calling the UDF from python ---------------------------------
-CREATE OR REPLACE FUNCTION create_model_arch_transfer_learning() RETURNS VOID AS $$
-from keras.layers import *
-from keras import Sequential
-import numpy as np
-import plpy
-
-model = Sequential()
-model.add(Conv2D(1, kernel_size=(1, 1), activation='relu', input_shape=(1,1,1,)))
-weights = model.get_weights()
-weights_flat = [ w.flatten() for w in weights ]
-weights1d = np.array([j for sub in weights_flat for j in sub])
-weights1d = np.ones_like(weights1d)
-weights_bytea = weights1d.tostring()
-
-load_query = plpy.prepare("""SELECT load_keras_model(
- 'test_keras_model_arch_table',
- $1, $2)
- """, ['json','bytea'])
-plpy.execute(load_query, [model.to_json(), weights_bytea])
-$$ LANGUAGE plpythonu VOLATILE;
-
-DROP TABLE IF EXISTS test_keras_model_arch_table;
-SELECT create_model_arch_transfer_learning();
-
-select assert(model_weights = '\000\000\200?\000\000\200?', 'loading weights from udf failed')
-from test_keras_model_arch_table;
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in
deleted file mode 100644
index 7c9ad5e..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in
+++ /dev/null
@@ -1,118 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
--------------------- CIFAR 10 test input tables -----------------
-
-DROP TABLE IF EXISTS cifar_10_sample;
-CREATE TABLE cifar_10_sample(id INTEGER, y SMALLINT, y_text TEXT, imgpath TEXT, x REAL[]);
-COPY cifar_10_sample FROM STDIN DELIMITER '|';
-0|0|'cat'|'0/img0.jpg'|{{{202,204,199},{202,204,199},{204,206,201},{206,208,203},{208,210,205},{209,211,206},{210,212,207},{212,214,210},{213,215,212},{215,217,214},{216,218,215},{216,218,215},{215,217,214},{216,218,215},{216,218,215},{216,218,214},{217,219,214},{217,219,214},{218,220,215},{218,219,214},{216,217,212},{217,218,213},{218,219,214},{214,215,209},{213,214,207},{212,213,206},{211,212,205},{209,210,203},{208,209,202},{207,208,200},{205,206,199},{203,204,198}},{{206,208,203},{206,208,203},{207,209,204},{209,211,207},{211,213,209},{212,214,210},{213,215,211},{215,217,214},{216,218,215},{218,220,217},{220,222,219},{220,222,219},{220,222,219},{220,222,220},{221,223,219},{220,222,218},{220,222,217},{221,223,218},{221,223,218},{220,222,217},{218,219,214},{215,216,211},{220,221,216},{220,221,215},{217,218,212},{215,216,210},{214,215,209},{213,214,208},{212,213,207},{210,211,205},{208,209,203},{206,207,201}},{{210,212,207},{211,213,208},{212,214,210},{214,216,213},{215,217,215},{217,219,216},{218,220,217},{219,221,219},{221,223,220},{221,223,220},{222,224,221},{223,225,222},{224,226,223},{226,228,225},{226,228,225},{226,228,224},{226,228,223},{227,229,224},{227,228,223},{226,228,223},{225,226,221},{221,222,217},{221,222,217},{223,224,219},{221,222,218},{219,220,216},{219,220,216},{218,219,214},{216,216,212},{214,214,210},{212,213,208},{210,211,206}},{{212,214,209},{213,215,210},{215,217,213},{217,219,217},{218,220,219},{220,222,221},{222,224,223},{224,226,223},{225,227,224},{226,228,225},{230,232,229},{230,232,229},{228,230,227},{229,231,228},{230,232,229},{231,233,229},{232,234,229},{233,235,230},{233,235,230},{233,235,229},{234,235,230},{232,233,228},{229,230,225},{228,229,226},{228,228,226},{226,226,224},{224,224,222},{222,222,220},{220,220,218},{218,218,216},{216,216,213},{213,214,209}},{{215,217,212},{216,218,213},{218,220,215},{219,221,219},{221,223,222},{222,224,223},{224,226,225},{225,227,224},{227,229,225},{224,226,223},{209,211,209},{223,224,222},{232,234,231},{232,234,231},{234,236,233},{235,237,232},{237,239,234},{237,239,235},{238,240,235},{240,241,237},{240,241,236},{240,241,236},{240,241,236},{238,239,236},{237,237,235},{236,236,234},{234,234,232},{231,231,229},{228,229,227},{225,225,223},{221,221,219},{218,219,215}},{{218,220,217},{218,220,216},{220,222,218},{221,223,221},{223,225,222},{225,227,224},{226,228,225},{226,228,223},{232,235,228},{183,185,183},{79,80,84},{155,157,160},{236,238,235},{235,237,232},{236,238,236},{239,241,237},{241,243,240},{244,245,244},{244,246,245},{241,245,243},{242,245,242},{243,245,240},{245,246,241},{246,246,242},{245,246,241},{245,246,242},{244,245,241},{243,244,241},{240,241,239},{236,238,235},{232,234,231},{228,230,228}},{{221,223,220},{221,223,220},{222,224,221},{224,226,223},{226,228,224},{230,232,229},{232,234,231},{231,233,229},{235,238,232},{170,172,171},{34,34,42},{92,92,101},{223,224,224},{238,240,234},{238,240,236},{235,237,234},{216,218,216},{194,195,196},{200,202,205},{237,242,242},{244,247,245},{245,246,242},{247,248,242},{248,249,244},{249,250,245},{250,251,246},{250,251,246},{250,252,248},{249,251,248},{247,249,246},{246,248,245},{243,245,243}},{{222,224,221},{223,225,222},{225,227,224},{227,229,226},{230,232,229},{187,189,186},{172,174,171},{227,229,228},{243,245,241},{179,181,180},{38,38,46},{47,47,60},{191,192,197},{240,243,237},{242,245,238},{186,188,185},{82,84,83},{63,64,67},{72,73,77},{184,188,189},{246,250,249},{245,246,244},{247,248,244},{249,250,245},{249,250,245},{250,251,246},{251,253,248},{252,254,251},{252,254,251},{252,254,251},{252,254,252},{252,254,253}},{{222,224,221},{223,225,222},{226,228,225},{229,231,228},{233,235,232},{158,161,157},{72,74,72},{107,108,110},{190,192,191},{183,184,184},{44,45,52},{26,25,39},{135,135,141},{241,243,239},{246,248,243},{152,154,152},{37,38,39},{46,47,51},{47,48,53},{96,100,103},{223,226,228},{248,249,248},{247,247,245},{249,250,245},{249,250,245},{250,251,246},{250,251,246},{250,252,249},{250,252,250},{251,253,250},{251,253,250},{251,253,252}},{{222,225,221},{222,224,221},{225,228,224},{229,231,228},{232,234,231},{220,222,219},{163,165,163},{81,81,86},{70,71,74},{98,100,100},{41,41,48},{23,23,35},{76,77,83},{202,204,203},{221,223,223},{124,125,126},{42,43,46},{52,53,58},{51,52,59},{51,54,59},{158,161,164},{248,249,250},{248,248,248},{249,250,246},{249,250,245},{250,251,246},{250,251,246},{250,252,248},{250,252,249},{250,252,249},{250,252,249},{250,252,251}},{{223,225,223},{223,225,223},{227,228,226},{230,232,229},{231,233,231},{234,236,233},{236,238,236},{195,196,200},{103,104,107},{38,39,40},{25,26,30},{32,33,41},{39,41,45},{81,82,85},{93,95,99},{70,71,74},{46,47,51},{48,49,54},{48,50,56},{42,46,51},{81,84,87},{219,220,222},{251,251,251},{249,250,245},{249,250,245},{250,251,246},{250,251,246},{250,252,248},{250,252,249},{249,251,248},{249,251,248},{249,251,250}},{{224,224,225},{225,225,226},{228,228,229},{230,231,230},{232,234,233},{233,235,233},{234,236,235},{235,237,234},{212,214,211},{125,127,125},{24,26,26},{16,18,21},{28,30,34},{35,37,41},{48,50,54},{50,52,56},{56,59,62},{52,55,59},{43,46,50},{47,53,58},{47,53,57},{145,148,149},{246,247,246},{250,251,246},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,252,247},{250,251,247},{250,251,247},{250,251,249}},{{224,224,226},{225,225,227},{228,228,230},{230,231,231},{232,234,233},{234,235,235},{235,237,235},{235,237,232},{241,243,238},{148,150,147},{19,21,21},{12,13,17},{12,13,17},{19,20,24},{29,30,34},{28,29,33},{51,52,56},{53,54,58},{35,37,41},{40,46,52},{57,63,67},{70,73,76},{168,168,170},{247,248,244},{251,252,247},{251,252,247},{251,252,247},{251,252,247},{252,253,248},{251,252,247},{251,252,247},{251,251,249}},{{224,224,226},{225,225,227},{228,228,230},{229,231,231},{232,234,232},{233,235,234},{235,237,236},{238,240,236},{226,228,224},{86,88,88},{11,12,14},{17,17,21},{13,12,17},{11,10,15},{10,9,14},{13,13,17},{22,22,26},{22,21,26},{21,21,26},{44,48,54},{46,51,56},{30,33,38},{70,73,77},{214,214,214},{253,253,250},{250,251,247},{251,252,247},{251,252,247},{251,252,247},{252,253,248},{252,253,248},{251,251,249}},{{224,224,226},{224,224,226},{226,226,228},{228,229,229},{229,231,230},{229,231,230},{229,231,230},{233,235,233},{208,210,208},{62,64,65},{10,10,14},{11,10,15},{11,10,15},{11,10,15},{9,8,13},{9,8,13},{9,7,12},{10,8,13},{14,13,18},{29,31,36},{27,30,36},{28,32,39},{45,49,56},{168,169,172},{252,252,252},{250,250,247},{250,251,246},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,251,249}},{{224,224,226},{224,224,226},{225,225,226},{225,226,226},{226,228,227},{225,227,226},{227,229,228},{230,232,230},{200,201,202},{57,58,62},{10,11,16},{11,9,14},{11,9,14},{14,12,17},{13,10,15},{9,7,12},{8,6,11},{10,8,13},{9,7,12},{22,21,26},{27,29,35},{38,42,51},{46,52,62},{115,115,122},{236,235,237},{251,251,248},{250,251,245},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,251,249}},{{223,224,225},{223,224,225},{227,227,229},{230,231,232},{230,231,232},{222,224,224},{191,193,193},{156,157,158},{139,140,142},{44,45,48},{9,10,15},{12,9,15},{13,10,15},{13,10,15},{15,11,17},{16,12,18},{11,7,13},{11,7,13},{9,6,12},{15,14,19},{25,27,33},{41,46,55},{52,58,69},{49,52,61},{165,166,170},{252,254,252},{250,251,246},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,251,249}},{{220,222,222},{220,222,221},{182,183,187},{158,159,163},{158,159,162},{121,122,126},{77,78,81},{40,41,45},{32,33,37},{14,15,19},{8,8,12},{10,8,12},{11,9,12},{11,9,12},{12,9,13},{14,12,17},{13,10,16},{9,6,12},{7,4,10},{8,7,12},{18,19,24},{32,35,42},{40,45,54},{32,40,46},{66,73,76},{171,175,176},{244,246,244},{251,252,248},{252,253,247},{252,253,247},{252,253,248},{252,252,250}},{{219,221,220},{206,208,206},{96,97,101},{39,39,46},{47,48,55},{35,35,42},{34,34,41},{30,30,35},{22,23,27},{19,20,24},{7,8,12},{8,6,9},{10,8,11},{9,7,10},{9,7,10},{8,6,10},{8,6,11},{6,4,9},{7,5,10},{9,8,13},{10,10,15},{19,21,27},{35,39,47},{48,56,62},{39,47,50},{59,64,66},{197,200,201},{253,254,251},{251,252,247},{252,253,248},{252,253,249},{253,253,252}},{{223,225,225},{186,188,186},{60,61,65},{25,25,33},{30,30,39},{35,35,44},{34,34,42},{31,32,37},{27,28,31},{25,26,30},{16,17,21},{7,6,9},{11,9,12},{11,9,12},{8,6,9},{7,5,9},{7,5,9},{6,4,8},{8,6,9},{12,10,13},{10,9,13},{10,10,16},{35,38,44},{47,54,59},{53,60,64},{46,51,55},{113,118,122},{241,242,241},{252,252,249},{251,251,249},{252,252,249},{252,252,251}},{{227,228,228},{198,200,198},{80,81,84},{39,39,45},{30,31,37},{23,24,30},{20,20,26},{23,24,28},{23,24,28},{24,25,29},{19,20,24},{10,8,12},{9,6,9},{11,9,12},{5,3,6},{6,4,6},{6,4,6},{6,4,6},{8,6,8},{13,9,11},{10,7,10},{11,10,15},{23,24,30},{36,39,44},{49,54,59},{52,59,65},{69,76,82},{212,213,214},{254,254,253},{250,250,249},{250,250,249},{251,251,250}},{{225,227,226},{223,225,223},{180,181,184},{150,152,154},{119,121,121},{78,80,80},{48,49,50},{26,27,31},{17,17,22},{13,13,18},{12,13,17},{14,13,16},{8,6,9},{6,4,7},{4,2,5},{5,3,4},{5,3,3},{4,2,3},{8,6,7},{14,10,10},{11,7,9},{12,10,14},{18,18,23},{24,25,30},{45,49,54},{60,68,76},{50,58,67},{163,165,167},{250,250,249},{249,249,249},{250,250,250},{250,250,250}},{{222,223,224},{222,224,223},{225,226,229},{226,228,228},{218,220,219},{199,201,200},{169,172,171},{133,135,137},{96,98,101},{62,64,67},{39,41,44},{29,29,31},{18,16,19},{8,6,9},{4,3,5},{4,2,4},{6,4,6},{7,5,6},{10,8,9},{13,9,10},{14,10,12},{16,14,17},{15,15,19},{20,20,25},{28,31,37},{44,49,57},{47,54,63},{96,100,105},{224,225,227},{251,251,252},{250,249,249},{250,249,250}},{{216,217,219},{211,212,214},{211,212,214},{212,214,215},{211,214,215},{208,210,211},{199,201,202},{185,187,189},{162,165,167},{128,130,132},{87,89,91},{63,63,65},{53,53,55},{43,43,45},{33,32,34},{24,24,26},{21,20,22},{18,17,19},{14,14,15},{9,8,10},{11,10,12},{13,13,15},{12,12,15},{25,25,31},{23,23,31},{20,21,28},{34,36,44},{65,71,81},{177,181,189},{251,251,254},{251,247,249},{250,249,250}},{{202,203,205},{183,184,186},{165,166,168},{147,148,150},{135,136,138},{138,139,141},{141,142,144},{140,141,143},{133,134,136},{118,119,121},{93,94,96},{73,73,75},{70,70,72},{69,69,71},{67,67,69},{62,62,64},{57,57,59},{63,63,65},{50,51,53},{42,42,44},{37,37,39},{24,24,26},{15,15,17},{17,18,23},{17,18,24},{12,13,19},{19,20,26},{37,42,52},{107,111,119},{229,228,232},{251,248,250},{248,247,249}},{{194,195,197},{168,169,171},{134,135,137},{94,94,96},{70,70,72},{68,68,70},{70,70,72},{71,71,73},{72,72,74},{72,72,74},{67,67,69},{59,59,61},{58,58,60},{60,60,62},{61,61,63},{62,62,64},{78,78,80},{92,92,94},{99,99,101},{102,102,104},{115,115,117},{121,121,123},{116,116,118},{104,104,109},{90,91,95},{75,76,81},{54,55,60},{62,64,71},{112,112,119},{212,211,215},{251,249,251},{247,246,248}},{{187,188,190},{160,161,163},{124,125,127},{84,83,86},{66,65,67},{62,61,64},{59,58,61},{58,57,59},{56,55,58},{54,54,56},{53,52,55},{52,52,54},{54,54,56},{56,56,58},{57,57,59},{60,60,62},{71,71,73},{77,77,79},{96,96,98},{105,105,107},{123,123,125},{150,150,152},{175,175,177},{192,193,196},{202,203,206},{202,203,206},{189,190,193},{194,194,198},{217,216,220},{239,237,241},{248,246,249},{247,247,249}},{{190,191,193},{160,162,163},{120,121,123},{81,79,82},{68,65,69},{65,62,65},{61,59,62},{58,56,59},{55,52,56},{52,50,53},{51,49,52},{49,49,51},{50,50,52},{51,52,54},{54,54,56},{54,54,56},{52,52,54},{58,58,60},{75,75,77},{92,92,94},{112,112,114},{136,136,138},{160,160,162},{183,183,185},{203,204,206},{224,225,227},{243,243,245},{251,248,250},{249,246,248},{246,244,247},{246,244,248},{246,245,248}},{{206,207,210},{184,185,188},{149,149,152},{104,102,105},{73,71,73},{64,62,64},{61,60,62},{60,57,60},{57,53,57},{55,51,55},{53,49,53},{51,49,52},{50,50,52},{50,49,52},{50,50,52},{48,48,50},{48,47,50},{53,52,54},{64,64,66},{84,83,85},{105,105,107},{129,129,131},{152,152,154},{174,175,177},{194,195,197},{213,214,216},{228,228,230},{239,237,239},{244,241,244},{244,243,246},{245,244,248},{245,244,247}},{{218,217,222},{210,209,214},{194,194,198},{161,162,162},{115,117,116},{79,80,79},{60,62,61},{57,58,59},{57,56,58},{57,52,56},{55,50,54},{52,49,53},{50,48,51},{49,48,51},{49,48,51},{50,49,52},{51,49,52},{54,52,55},{61,59,62},{78,77,79},{98,98,100},{120,120,122},{143,143,145},{166,166,168},{187,187,189},{208,208,210},{224,224,226},{236,235,239},{241,240,245},{243,242,247},{244,242,247},{243,243,247}},{{219,218,223},{217,216,221},{216,216,220},{206,207,208},{182,184,184},{144,146,145},{100,102,102},{70,71,72},{57,56,58},{54,51,54},{54,49,53},{51,49,52},{49,48,51},{49,47,50},{50,48,51},{52,50,53},{54,52,55},{55,53,56},{59,57,60},{72,72,74},{92,92,94},{111,111,113},{133,133,135},{156,156,158},{179,179,181},{200,200,202},{217,217,219},{230,230,234},{238,237,242},{241,240,245},{241,240,245},{241,240,245}},{{217,216,221},{216,215,220},{217,216,221},{216,217,219},{212,213,214},{198,199,201},{170,171,172},{131,133,135},{94,95,97},{69,68,70},{58,55,58},{53,51,54},{52,50,53},{52,50,53},{51,49,52},{52,50,53},{54,52,55},{55,53,56},{59,57,60},{69,68,70},{86,86,88},{104,104,106},{125,125,127},{147,147,149},{172,172,174},{193,193,195},{210,210,212},{224,223,227},{233,232,237},{239,238,243},{239,238,243},{240,238,243}}}
-1|1|'dog'|'0/img2.jpg'|{{{126,118,110},{122,115,108},{126,119,111},{127,119,109},{130,122,111},{130,122,111},{132,124,113},{133,125,114},{130,122,111},{132,124,113},{134,126,115},{131,123,112},{131,123,112},{134,126,115},{133,125,114},{136,128,117},{137,129,118},{137,129,118},{136,128,117},{131,123,112},{130,122,111},{132,124,113},{132,124,113},{132,124,113},{129,122,110},{127,121,109},{127,121,109},{125,119,107},{124,118,106},{124,118,106},{120,114,102},{117,111,99}},{{122,115,107},{119,112,104},{121,114,106},{124,116,107},{129,121,110},{130,122,111},{130,122,111},{130,122,111},{128,120,109},{131,123,112},{134,126,115},{132,124,113},{132,124,113},{134,126,115},{136,128,117},{139,131,120},{141,133,122},{138,130,119},{135,127,116},{130,122,111},{131,123,112},{132,124,113},{133,125,114},{133,125,114},{131,124,113},{129,123,111},{130,124,112},{129,123,111},{127,121,109},{124,118,106},{122,116,104},{117,111,99}},{{122,115,107},{121,114,106},{121,114,106},{125,118,108},{128,120,109},{129,121,110},{131,123,112},{130,122,111},{128,120,109},{131,123,112},{133,125,114},{134,126,115},{136,128,117},{137,129,118},{138,130,119},{139,131,120},{139,131,120},{139,131,120},{139,131,120},{139,131,120},{139,131,120},{137,129,118},{137,129,118},{138,130,119},{136,129,118},{135,129,117},{133,127,115},{130,124,112},{129,123,111},{126,120,108},{123,117,105},{121,115,103}},{{123,116,108},{123,116,108},{125,118,110},{127,119,109},{127,119,108},{129,121,110},{132,125,113},{131,123,112},{130,122,111},{134,127,115},{134,126,115},{135,127,116},{133,125,114},{134,126,116},{136,128,118},{140,132,121},{142,134,123},{141,133,122},{141,132,122},{142,133,123},{141,132,122},{141,133,122},{140,131,120},{137,128,117},{136,128,117},{136,130,118},{132,127,115},{128,122,110},{127,121,109},{127,121,109},{125,119,107},{123,117,105}},{{124,119,110},{125,119,111},{128,122,113},{129,123,113},{132,123,113},{136,123,115},{137,125,116},{134,125,116},{135,125,116},{136,125,116},{134,125,115},{133,126,116},{136,124,115},{135,127,113},{137,131,115},{140,130,118},{141,130,120},{139,131,120},{137,132,118},{135,133,117},{134,130,117},{136,130,118},{136,130,119},{134,129,117},{132,127,115},{132,126,114},{131,125,113},{131,124,112},{129,122,110},{127,121,109},{128,122,110},{128,122,110}},{{124,121,112},{124,121,112},{128,124,115},{129,125,116},{136,124,118},{145,124,121},{143,126,121},{136,128,120},{136,126,119},{140,123,118},{136,126,118},{133,129,121},{146,130,130},{136,130,115},{135,133,112},{138,131,117},{138,130,119},{139,132,120},{137,132,117},{133,132,113},{132,131,115},{135,130,118},{136,129,118},{135,130,118},{131,127,115},{132,124,113},{132,124,113},{131,123,112},{129,122,110},{128,122,110},{127,121,109},{127,121,109}},{{123,120,111},{122,119,110},{124,121,112},{126,122,113},{132,122,115},{138,125,119},{135,128,120},{128,130,119},{130,129,119},{134,124,117},{127,118,110},{129,126,120},{142,135,139},{154,139,136},{141,130,123},{130,133,125},{131,135,121},{138,135,122},{143,130,122},{145,124,120},{142,126,119},{140,128,118},{141,128,119},{142,126,118},{139,125,116},{134,126,115},{134,126,115},{133,125,114},{131,123,112},{129,122,110},{126,120,108},{124,118,106}},{{121,118,109},{121,118,109},{122,119,110},{123,118,109},{127,121,113},{130,126,117},{131,131,121},{134,136,125},{130,131,121},{133,130,121},{133,127,119},{136,131,123},{115,104,98},{128,91,93},{151,118,117},{139,136,125},{135,137,122},{140,136,124},{139,132,124},{141,131,126},{140,128,121},{136,129,117},{134,131,118},{135,128,117},{137,126,116},{135,127,116},{135,127,116},{133,125,114},{131,123,112},{128,122,110},{127,121,109},{125,119,107}},{{124,121,112},{124,121,112},{126,123,114},{127,122,113},{128,124,115},{132,130,121},{138,131,123},{162,149,143},{156,144,138},{137,133,124},{141,136,128},{143,136,125},{137,120,105},{119,80,78},{144,108,108},{147,136,125},{142,133,123},{146,133,124},{149,134,123},{148,132,120},{151,130,121},{140,129,119},{127,134,118},{126,133,117},{131,128,116},{135,127,116},{134,126,115},{132,124,113},{130,122,111},{128,122,110},{128,122,110},{126,120,108}},{{126,123,114},{126,123,114},{128,125,116},{130,127,118},{129,129,119},{135,131,122},{143,122,119},{142,106,108},{163,131,132},{147,134,128},{140,136,127},{139,132,124},{141,134,128},{120,115,122},{134,131,142},{151,151,155},{143,139,133},{154,128,121},{152,90,85},{155,61,56},{158,77,80},{155,115,115},{141,131,121},{133,131,118},{135,128,117},{135,127,116},{134,126,115},{132,124,113},{130,123,112},{129,123,111},{126,120,108},{126,120,108}},{{126,121,112},{126,121,113},{130,125,116},{131,126,116},{131,128,117},{134,127,119},{141,123,120},{110,64,71},{121,57,65},{151,117,110},{143,135,122},{132,134,122},{136,138,130},{102,114,121},{80,97,121},{90,107,138},{130,146,164},{149,133,146},{142,71,84},{165,54,64},{151,34,43},{150,63,65},{159,116,110},{142,131,119},{134,130,118},{135,127,116},{136,128,117},{134,126,115},{131,123,112},{130,124,112},{126,120,108},{124,118,106}},{{129,122,114},{128,121,114},{131,124,116},{133,127,115},{135,129,117},{136,129,120},{138,132,127},{150,123,127},{144,96,100},{152,120,109},{144,129,114},{131,128,119},{171,174,177},{160,168,180},{132,146,167},{97,117,145},{106,131,159},{136,135,162},{150,112,134},{186,129,141},{202,129,128},{149,78,70},{159,118,107},{146,134,121},{135,132,119},{136,128,117},{136,128,117},{136,128,117},{134,126,115},{134,128,116},{131,125,113},{126,120,108}},{{133,126,118},{132,125,117},{134,127,119},{136,130,118},{136,130,118},{138,132,123},{139,131,126},{113,113,117},{126,135,146},{168,171,175},{149,142,143},{119,111,122},{148,157,188},{194,207,232},{166,178,191},{159,172,180},{180,194,207},{191,192,207},{170,156,167},{189,168,173},{211,196,191},{149,139,127},{147,135,123},{149,136,123},{145,134,122},{140,132,121},{138,130,119},{136,128,117},{132,125,113},{132,126,114},{131,125,113},{128,122,110}},{{134,127,119},{135,128,120},{136,128,121},{136,130,118},{137,131,119},{137,131,122},{142,134,128},{90,93,108},{33,52,92},{89,107,144},{145,161,193},{152,169,196},{116,138,164},{133,151,175},{167,177,200},{189,194,216},{207,211,225},{210,216,225},{204,212,217},{203,212,214},{173,170,171},{143,132,131},{146,137,129},{145,137,122},{144,136,120},{141,133,122},{139,131,120},{138,130,119},{133,125,114},{130,124,112},{128,122,110},{126,120,108}},{{131,124,116},{134,128,120},{135,128,120},{136,130,118},{139,133,121},{138,131,122},{109,101,95},{121,117,130},{103,109,150},{52,67,114},{60,87,130},{108,140,172},{150,174,198},{146,165,191},{128,141,172},{140,148,181},{158,164,187},{169,181,196},{183,201,208},{183,195,199},{154,140,144},{156,133,132},{150,137,128},{142,138,121},{141,138,121},{141,133,122},{137,129,118},{136,128,117},{135,127,116},{133,127,115},{129,123,111},{128,122,110}},{{132,124,116},{130,121,114},{135,126,120},{140,131,122},{140,134,122},{138,135,124},{125,119,115},{157,159,162},{162,174,189},{140,149,179},{140,154,186},{131,146,177},{99,114,155},{137,152,188},{164,182,209},{164,182,203},{156,170,195},{146,162,184},{147,161,179},{168,173,191},{159,159,158},{147,144,128},{146,139,124},{146,139,123},{143,135,123},{142,134,124},{136,128,117},{133,125,114},{134,127,116},{127,121,109},{129,123,111},{129,123,111}},{{141,129,119},{136,122,116},{135,124,120},{141,129,128},{135,130,122},{148,152,145},{185,191,204},{141,169,193},{119,147,170},{181,181,202},{192,200,212},{204,203,220},{167,172,199},{132,143,172},{119,136,168},{137,160,193},{169,188,213},{185,192,204},{175,177,202},{92,107,179},{97,120,166},{134,143,141},{149,140,126},{147,137,121},{140,132,126},{140,131,128},{138,131,116},{137,129,115},{137,128,119},{133,125,115},{131,124,112},{129,122,111}},{{139,129,117},{132,130,117},{127,134,120},{131,137,124},{157,154,155},{180,178,199},{118,136,175},{89,115,149},{153,146,163},{192,179,185},{182,207,197},{193,217,214},{202,206,219},{193,200,218},{171,186,209},{129,149,182},{96,125,171},{137,161,179},{169,183,203},{53,62,153},{34,44,157},{102,112,172},{124,127,141},{145,138,133},{143,133,125},{140,133,117},{139,132,117},{137,129,117},{136,128,116},{137,129,118},{135,127,116},{130,122,111}},{{134,128,114},{137,129,112},{143,130,112},{145,90,76},{165,123,127},{107,118,147},{94,115,158},{162,137,167},{152,76,85},{165,113,119},{207,202,202},{200,212,217},{213,213,226},{223,226,236},{220,229,236},{210,224,231},{150,175,200},{71,107,153},{104,138,195},{107,119,184},{50,51,159},{70,83,198},{60,77,162},{114,111,149},{140,126,118},{136,130,103},{133,125,112},{136,127,117},{134,126,110},{135,127,115},{134,126,115},{129,121,110}},{{137,127,114},{154,108,97},{159,66,64},{168,25,19},{151,29,30},{117,61,76},{168,148,162},{201,168,175},{180,136,140},{192,168,177},{222,207,225},{228,224,240},{219,216,222},{187,186,189},{189,190,189},{214,215,210},{233,229,229},{181,198,221},{77,114,170},{93,124,184},{133,149,200},{68,82,181},{33,46,185},{86,86,176},{134,125,146},{137,128,123},{136,128,117},{135,128,113},{134,126,112},{132,124,113},{131,123,112},{129,121,110}},{{139,122,111},{133,65,55},{151,14,14},{167,15,13},{174,17,17},{171,20,23},{155,53,47},{191,142,135},{216,219,216},{213,237,237},{214,218,224},{166,172,170},{95,92,90},{46,40,39},{51,45,44},{82,74,75},{125,112,115},{164,166,160},{147,167,173},{55,80,130},{118,138,172},{139,150,196},{64,67,163},{104,112,203},{113,111,187},{121,103,148},{140,131,120},{136,129,106},{133,124,120},{130,122,113},{129,121,110},{127,119,108}},{{140,121,112},{133,85,73},{137,45,37},{147,29,31},{162,20,19},{180,18,10},{173,19,14},{161,56,61},{181,157,159},{148,152,149},{90,79,74},{44,39,25},{55,47,39},{69,59,54},{57,45,43},{43,31,31},{33,28,24},{34,29,26},{54,48,52},{58,57,67},{73,79,100},{142,144,158},{137,137,155},{113,126,182},{84,89,183},{102,86,158},{130,123,125},{135,128,110},{135,123,122},{131,121,111},{128,120,107},{126,118,106}},{{135,124,117},{106,91,83},{65,46,39},{114,58,55},{154,47,45},{162,31,28},{145,27,26},{99,25,26},{61,33,30},{39,31,22},{52,47,36},{97,89,77},{135,124,111},{143,132,119},{137,125,113},{126,114,102},{110,99,92},{93,81,78},{74,60,59},{63,51,47},{80,71,69},{114,110,107},{132,133,133},{129,139,151},{141,151,187},{166,169,211},{173,172,189},{140,133,131},{134,123,112},{133,123,107},{129,120,104},{127,119,104}},{{129,120,112},{68,60,52},{19,12,5},{33,27,22},{73,42,41},{81,26,28},{50,12,14},{32,15,11},{51,37,28},{92,85,73},{133,125,114},{151,137,127},{147,135,121},{144,132,118},{147,135,121},{149,137,123},{150,138,125},{148,136,124},{145,133,120},{137,125,113},{115,105,92},{85,79,69},{101,101,95},{128,133,130},{159,169,173},{199,211,219},{225,231,232},{184,185,176},{133,129,114},{129,122,107},{127,119,105},{125,117,104}},{{128,120,109},{109,101,91},{76,67,58},{25,33,23},{14,17,12},{30,18,16},{45,42,38},{78,81,71},{120,115,100},{140,131,118},{144,128,117},{145,126,116},{144,132,118},{144,132,118},{145,133,119},{145,133,119},{142,131,114},{145,134,115},{147,136,118},{148,137,119},{146,137,118},{128,122,105},{97,93,79},{90,89,78},{116,120,113},{155,163,161},{200,206,205},{220,225,226},{178,181,181},{127,125,116},{122,118,107},{124,117,104}},{{125,117,104},{124,116,104},{129,122,109},{99,95,79},{72,63,51},{93,76,70},{126,111,100},{136,130,113},{136,133,114},{142,128,113},{135,116,103},{129,112,99},{130,118,104},{137,125,111},{142,130,116},{144,132,118},{142,131,113},{144,133,114},{145,134,115},{145,134,115},{144,134,117},{144,135,119},{142,134,118},{112,104,89},{79,72,64},{96,92,94},{147,146,154},{194,197,211},{222,228,241},{170,174,170},{119,119,110},{123,117,104}},{{124,117,101},{120,113,97},{125,118,103},{131,121,104},{130,120,109},{136,124,119},{143,126,117},{139,127,110},{139,132,113},{146,129,114},{142,125,109},{132,123,105},{134,123,109},{141,129,115},{142,130,116},{143,131,117},{141,129,114},{144,132,117},{145,133,118},{144,132,117},{140,128,116},{140,128,117},{144,131,119},{145,131,118},{131,115,104},{99,84,77},{86,77,74},{112,112,114},{166,172,176},{203,211,209},{153,156,149},{120,115,103}},{{124,117,103},{122,115,101},{130,122,109},{130,122,109},{127,122,112},{130,125,118},{134,125,116},{138,128,114},{140,130,113},{142,127,112},{140,126,111},{138,129,113},{142,130,116},{139,127,113},{134,122,108},{142,130,116},{141,129,116},{142,130,117},{144,132,119},{143,131,118},{133,122,111},{136,126,116},{140,128,118},{139,127,116},{141,127,113},{140,126,110},{119,110,96},{92,88,75},{81,81,71},{120,120,114},{156,154,145},{123,117,105}},{{126,117,106},{128,120,109},{130,122,111},{129,121,110},{131,122,111},{132,124,113},{133,125,114},{136,126,114},{139,127,113},{138,126,112},{138,126,112},{140,128,114},{139,127,113},{137,125,111},{135,123,109},{142,130,116},{140,128,114},{140,128,114},{140,128,114},{140,128,114},{136,126,114},{136,128,117},{137,129,118},{137,129,118},{134,126,114},{134,127,111},{137,130,115},{136,128,113},{119,112,97},{97,89,78},{113,105,93},{121,113,102}},{{126,118,107},{126,118,107},{128,120,109},{130,122,111},{130,122,111},{131,123,112},{134,126,115},{136,126,114},{137,125,111},{137,125,111},{136,124,110},{139,127,113},{139,127,113},{139,127,113},{137,125,111},{138,126,112},{138,126,112},{138,126,112},{139,127,113},{139,127,113},{135,125,113},{138,130,119},{142,134,123},{141,133,122},{140,132,120},{138,130,117},{138,130,117},{135,127,114},{138,130,118},{133,125,114},{122,114,103},{114,106,95}},{{125,117,106},{126,118,107},{127,119,108},{129,121,110},{129,121,110},{129,121,110},{133,125,114},{133,123,111},{134,122,108},{135,123,109},{136,124,110},{138,126,112},{138,126,112},{136,124,110},{135,123,109},{136,124,110},{136,124,110},{137,125,111},{139,127,113},{138,126,112},{136,127,115},{138,130,119},{142,134,123},{140,132,121},{138,130,119},{134,126,115},{135,127,116},{135,127,116},{132,124,113},{128,120,109},{121,113,102},{114,106,95}},{{123,115,104},{123,115,104},{126,118,107},{128,120,109},{129,121,110},{129,121,110},{131,123,112},{132,122,110},{133,121,107},{135,123,109},{135,123,109},{135,123,109},{136,124,110},{133,121,107},{134,122,108},{136,124,110},{136,124,110},{137,125,111},{134,122,108},{130,118,104},{133,123,111},{133,125,115},{136,128,117},{136,128,117},{135,127,117},{134,125,116},{132,123,114},{131,122,113},{129,121,111},{128,119,109},{126,118,107},{121,113,102}}}
-\.
-
-DROP TABLE IF EXISTS cifar_10_sample_batched;
-DROP TABLE IF EXISTS cifar_10_sample_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_batched','y','x', 1, 255);
-
-DROP TABLE IF EXISTS cifar_10_sample_val, cifar_10_sample_val_summary;
-SELECT validation_preprocessor_dl('cifar_10_sample','cifar_10_sample_val','y','x', 'cifar_10_sample_batched', 1);
---- NOTE: In order to test fit_merge, we need at least 2 rows in the batched table (1 on each segment).
-
-DROP TABLE IF EXISTS cifar_10_sample_int_batched;
-DROP TABLE IF EXISTS cifar_10_sample_int_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, 5);
-
--- This table is for testing a different input shape (3, 32, 32) instead of (32, 32, 3).
--- Create a table with image shape 3, 32, 32
-drop table if exists cifar_10_sample_test_shape;
-create table cifar_10_sample_test_shape(id INTEGER, y SMALLINT, x REAL[] );
-copy cifar_10_sample_test_shape from stdin delimiter '|';
-1|0|{{{248,248,250,245,245,246,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245},{247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245},{245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247},{248,248,250,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245},{247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245},{245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247},{249,249,251,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246},{248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248},{249,249,251,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246},{248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248},{249,249,251,245,245,247,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,249,246,246,249,246,246},{249,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,246,249,246,246,249,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248},{249,249,251,245,245,247,246,246,248,247,246,249,247,246,248,246,246,247,245,246,248,246,246,251,247,245,252,247,245,251,247,245},{250,247,245,250,248,245,249,247,245,248,246,243,247,246,243,247,245,244,248,244,244,248,246,245,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,245,249,246,246,249,245,246,249,245,246,249,246,246,247,246,246,246,246,246,247},{249,249,251,245,245,247,246,246,248,247,246,248,247,246,248,247,246,249,246,246,250,245,246,250,245,246,248,245,246,247,246,246},{247,246,246,247,247,246,247,246,246,247,245,244,245,245,244,245,244,244,247,244,243,247,245,245,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,246,246,246,244,246,247,244,246,247,245,245,246,247,246,245,250,246,246,248,246,246,248},{249,249,251,245,245,247,246,246,248,244,245,245,242,243,246,244,243,248,246,244,247,247,247,246,248,250,247,247,249,247,246,247},{248,245,246,248,245,246,251,245,246,251,243,244,249,243,244,248,244,244,244,244,244,244,245,245,247,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,247,246,247,245,246,246,244,247,245,243,249,247,246,246,245,249,246,245,251,246,245,250},{249,249,251,245,245,247,245,245,247,245,248,249,245,248,251,245,245,247,247,246,237,245,243,229,235,233,226,238,236,233,243,240},{241,248,245,249,246,245,253,246,245,253,245,244,252,245,245,251,245,246,243,245,246,241,246,246,246,246,245,249,246,246,249,246},{246,248,246,246,248,246,246,248,245,245,250,244,244,251,248,245,249,247,242,239,240,235,227,242,243,235,245,245,247,246,245,251},{249,249,251,244,244,246,233,233,235,187,192,193,161,166,169,160,161,157,208,207,179,230,227,188,215,210,179,215,209,184,231,224},{205,240,233,220,239,236,226,241,238,229,242,238,230,241,238,229,238,239,229,244,245,238,246,246,245,246,246,250,246,246,249,246},{246,248,247,246,248,247,246,248,245,245,250,246,247,252,240,238,232,222,216,193,202,198,164,206,209,180,241,242,235,246,245,250},{248,248,250,245,245,247,231,231,233,127,131,133,39,44,46,26,27,17,109,109,70,169,167,108,180,174,116,191,183,131,203,194},{147,184,175,133,145,139,102,149,144,106,150,146,108,151,148,114,186,187,169,240,241,234,245,246,245,246,245,250,246,246,251,245}},{{246,250,245,246,249,245,246,249,246,248,248,229,231,219,196,198,164,196,195,140,185,183,121,186,191,148,240,242,229,247,246,250},{249,249,251,244,244,246,244,244,246,233,232,234,174,173,172,114,113,100,112,112,85,112,110,70,112,108,59,125,119,67,151,143},{86,149,139,83,124,113,60,111,103,42,115,111,51,151,148,103,203,204,184,241,243,237,246,247,249,247,247,251,245,247,252,246},{246,251,248,247,249,240,243,241,216,219,204,196,197,157,195,200,138,197,199,132,178,178,118,208,210,176,246,245,244,245,245,250},{249,249,251,244,244,246,244,244,246,246,244,246,249,247,245,239,238,230,222,221,209,199,197,178,124,120,91,82,77,36,132,126},{71,171,164,102,171,162,100,143,137,68,147,143,80,194,191,145,201,204,180,211,214,204,227,229,226,236,238,235,245,243,239,249},{242,236,235,224,212,209,198,177,194,183,153,187,176,134,165,158,106,141,137,80,109,107,52,187,187,159,248,246,251,245,245,249},{248,248,250,244,244,246,244,244,246,245,244,244,242,242,241,243,243,242,250,249,249,253,251,251,189,185,176,91,87,61,96,92},{46,155,152,93,153,150,88,149,147,82,153,151,92,177,175,130,197,200,172,199,203,187,186,190,179,166,169,156,153,152,138,143},{133,117,115,96,73,123,102,69,145,124,86,132,108,74,90,68,36,80,68,26,100,98,47,213,212,190,249,247,251,244,245,248},{247,247,249,244,244,246,244,244,246,244,244,245,244,244,245,244,244,247,242,242,248,243,242,249,244,241,245,179,176,165,104,101},{73,82,80,40,89,90,43,101,102,53,108,107,60,120,118,78,130,134,105,144,149,131,161,165,150,154,159,140,118,124,106,87},{86,70,74,64,42,69,58,29,74,60,27,81,58,27,84,60,28,100,86,48,156,155,114,223,222,207,247,245,250,245,246,246},{247,247,249,244,243,246,244,244,246,243,244,247,242,245,247,242,245,247,242,245,247,243,245,247,244,244,248,241,240,240,219,216},{209,140,138,125,85,88,68,79,81,56,86,85,54,83,81,46,62,65,40,55,60,44,77,81,66,91,96,75,106,113,89,109},{112,90,104,100,78,88,87,63,72,72,45,70,62,26,94,79,29,142,132,87,149,149,121,205,203,195,246,244,250,244,245,244},{246,247,248,243,243,244,244,244,245,242,244,247,241,245,247,241,245,245,241,245,243,241,245,243,242,244,245,244,244,248,252,249},{254,212,210,213,185,186,184,160,161,153,159,159,140,107,107,79,61,64,42,49,53,38,41,45,31,38,42,25,38,41,18,58},{56,29,76,72,43,83,83,52,82,86,58,74,75,37,139,133,71,173,169,114,129,131,104,202,201,197,247,245,250,242,244,243},{245,247,245,242,243,242,242,244,243,242,244,245,242,243,245,242,243,245,242,244,244,242,243,244,242,243,245,242,243,246,242,242},{246,175,175,177,159,160,154,142,143,133,133,135,120,105,108,89,89,96,75,85,93,75,72,78,64,56,63,51,42,45,34,47},{47,26,55,55,18,67,65,21,79,74,38,74,68,30,117,113,58,176,174,122,138,137,108,177,178,176,247,247,250,243,244,245},{245,247,246,242,243,242,242,244,243,242,243,244,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,243,244},{245,211,213,212,210,215,207,178,181,171,133,134,122,108,107,94,89,97,80,76,88,69,77,89,69,80,91,72,63,69,51,68},{70,46,72,72,40,99,96,65,101,96,59,62,57,8,69,66,21,132,127,103,203,198,196,228,229,231,243,244,246,242,243,245},{245,246,248,241,243,244,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,241,242},{244,245,248,248,243,252,246,245,249,242,239,237,230,218,212,205,194,193,184,133,135,122,60,62,43,76,80,54,82,85,57,79},{81,55,117,117,97,201,199,188,171,168,147,95,92,54,130,128,99,212,209,204,245,241,252,245,245,248,242,243,245,241,242,244},{244,245,249,241,242,246,241,242,246,241,242,245,242,243,245,241,242,244,241,242,244,241,242,244,241,242,244,241,242,244,241,242},{244,239,242,243,235,244,242,240,244,241,240,238,235,192,184,181,182,170,169,152,140,136,53,42,30,68,57,38,93,90,66,139},{141,122,214,213,207,244,244,245,228,226,225,210,207,200,225,223,215,244,244,240,242,243,243,242,243,245,243,244,246,243,244,246},{243,244,249,241,241,246,241,242,247,241,242,245,241,242,243,241,242,244,241,242,244,241,242,244,241,242,244,241,242,244,241,242}},{{244,241,242,245,241,242,246,242,243,245,243,242,242,221,219,217,214,205,205,196,185,183,100,90,83,121,112,100,188,184,171,234},{232,228,245,245,249,238,242,244,243,245,248,248,245,253,244,243,247,240,243,241,240,244,239,242,243,244,242,243,245,242,243,245},{243,244,250,240,241,247,240,242,247,240,241,246,240,241,245,240,242,245,241,242,245,241,242,245,241,242,245,241,242,245,241,242},{245,241,242,245,242,241,247,242,241,246,242,243,245,246,247,248,248,248,248,246,245,244,209,209,207,225,225,221,247,246,244,244},{242,247,240,240,248,239,243,246,239,242,243,241,240,245,241,241,247,240,243,247,239,243,245,241,242,245,241,242,245,241,242,245},{241,244,251,238,241,247,238,241,248,238,241,248,239,241,248,239,241,248,239,242,248,239,241,247,240,241,246,240,241,246,241,242},{246,241,241,246,240,241,246,240,241,246,241,242,246,240,241,245,239,240,245,240,241,245,242,243,247,241,243,246,240,240,245,240},{240,246,241,241,247,240,241,246,240,242,246,241,242,246,241,242,247,241,242,247,241,242,247,241,242,247,241,242,247,241,242,247},{240,243,250,237,240,247,238,241,248,238,241,248,237,240,247,237,240,247,237,240,247,238,241,247,240,241,246,240,241,246,239,240},{245,239,240,245,240,241,246,240,241,246,239,240,245,239,241,246,240,241,246,240,241,247,239,240,245,240,241,246,241,242,247,240},{241,246,240,241,246,240,241,246,240,241,246,240,241,246,240,241,246,241,242,247,241,242,247,241,242,247,241,242,247,241,242,247},{239,242,249,237,240,247,238,240,247,237,240,247,236,239,246,236,239,246,236,239,246,237,240,246,238,241,246,237,241,245,237,240},{244,237,240,245,238,241,246,238,241,246,237,240,245,237,240,245,238,241,245,238,241,246,238,241,246,238,241,246,238,241,246,238},{241,246,238,241,246,238,241,246,238,241,246,238,241,245,238,241,246,239,242,247,239,242,247,239,242,247,239,242,247,239,242,247},{239,242,249,236,239,246,236,239,246,237,240,247,237,240,247,236,239,246,236,239,246,236,240,246,235,241,245,235,241,245,235,241},{245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,234,241,244,235,241,245,235},{242,245,236,242,246,236,242,246,235,242,245,234,240,244,235,241,245,236,242,246,236,242,246,236,242,246,236,242,246,236,242,246},{239,242,249,236,239,245,236,239,246,236,239,246,236,239,246,236,239,246,236,239,246,235,240,245,233,241,244,233,241,244,233,241},{244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233},{241,244,234,242,245,235,243,246,234,242,245,233,241,244,234,242,245,235,243,246,235,243,246,235,243,246,235,243,246,235,243,246},{237,242,249,234,239,246,235,239,246,234,239,246,234,238,246,235,239,246,235,239,246,234,239,246,232,240,245,232,240,244,232,240},{244,232,240,244,232,240,244,232,240,244,232,240,244,232,240,244,232,240,244,232,240,245,233,240,245,233,241,245,233,240,245,232},{240,245,233,241,246,234,242,247,234,241,246,233,240,245,233,241,245,234,242,246,234,242,246,234,242,247,234,242,247,234,242,247},{235,242,250,232,238,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239},{247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,233,240,247,233,240,248,233,240,247,233},{240,247,233,240,248,233,240,248,233,240,247,232,239,247,232,239,247,232,239,247,232,239,247,233,240,248,233,240,248,233,240,248},{235,242,250,231,239,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239},{247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232},{239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,233,240,248},{235,242,250,232,238,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239},{247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,231,238,246,231,238,246,231},{238,246,231,238,246,231,238,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,231,238,246,232,239,246,232,239,247}}}
-\.
-
-DROP TABLE IF EXISTS cifar_10_sample_test_shape_batched;
-DROP TABLE IF EXISTS cifar_10_sample_test_shape_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample_test_shape','cifar_10_sample_test_shape_batched','y','x', NULL, 255, 3);
-
-DROP TABLE IF EXISTS model_arch;
-SELECT load_keras_model('model_arch',
- $${
- "class_name": "Sequential",
- "keras_version": "2.1.6",
- "config": [{
- "class_name": "Conv2D", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}},
- "name": "conv2d_1",
- "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null,
- "dtype": "float32", "activation": "relu", "trainable": true,
- "data_format": "channels_last", "filters": 32, "padding": "valid",
- "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null,
- "bias_initializer": {"class_name": "Zeros", "config": {}},
- "batch_input_shape": [null, 32, 32, 3], "use_bias": true,
- "activity_regularizer": null, "kernel_size": [3, 3]}},
- {"class_name": "MaxPooling2D", "config": {"name": "max_pooling2d_1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}},
- {"class_name": "Dropout", "config": {"rate": 0.25, "noise_shape": null, "trainable": true, "seed": null, "name": "dropout_1"}},
- {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten_1", "data_format": "channels_last"}},
- {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer":
- {"class_name": "Zeros", "config": {}}, "units": 2, "use_bias": true, "activity_regularizer": null}
- }], "backend": "tensorflow"}$$);
-
-SELECT load_keras_model('model_arch',
- $${
- "class_name": "Sequential",
- "keras_version": "2.1.6",
- "config": [{
- "class_name": "Conv2D", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}},
- "name": "conv2d_1",
- "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null,
- "dtype": "float32", "activation": "relu", "trainable": true,
- "data_format": "channels_last", "filters": 32, "padding": "valid",
- "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null,
- "bias_initializer": {"class_name": "Zeros", "config": {}},
- "batch_input_shape": [null, 32, 32, 3], "use_bias": true,
- "activity_regularizer": null, "kernel_size": [3, 3]}},
- {"class_name": "MaxPooling2D", "config": {"name": "max_pooling2d_1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}},
- {"class_name": "Dropout", "config": {"rate": 0.25, "noise_shape": null, "trainable": true, "seed": null, "name": "dropout_1"}},
- {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten_1", "data_format": "channels_last"}},
- {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer":
- {"class_name": "Zeros", "config": {}}, "units": 5, "use_bias": true, "activity_regularizer": null}
- }], "backend": "tensorflow"}$$);
-
-SELECT load_keras_model('model_arch',
- $${
- "class_name": "Sequential",
- "keras_version": "2.1.6",
- "config": [{
- "class_name": "Conv2D", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}},
- "name": "conv2d_1",
- "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null,
- "dtype": "float32", "activation": "relu", "trainable": true,
- "data_format": "channels_first", "filters": 32, "padding": "valid",
- "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null,
- "bias_initializer": {"class_name": "Zeros", "config": {}},
- "batch_input_shape": [null, 3, 32, 32], "use_bias": true,
- "activity_regularizer": null, "kernel_size": [3, 3]}},
- {"class_name": "MaxPooling2D", "config": {"name": "max_pooling2d_1", "trainable": true, "data_format": "channels_first", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}},
- {"class_name": "Dropout", "config": {"rate": 0.25, "noise_shape": null, "trainable": true, "seed": null, "name": "dropout_1"}},
- {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten_1", "data_format": "channels_first"}},
- {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer":
- {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}
- }], "backend": "tensorflow"}$$);
-
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_evaluate.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_evaluate.sql_in
deleted file mode 100644
index cdda44a..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_evaluate.sql_in
+++ /dev/null
@@ -1,117 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-m4_include(`SQLCommon.m4')
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_cifar.setup.sql_in'
-)
-
--- -- Please do not break up the compile_params string
--- -- It might break the assertion
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_saved_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
-
--- Test that evaluate works as expected:
-DROP TABLE IF EXISTS evaluate_out;
-SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out', FALSE);
-
-SELECT assert(loss >= 0 AND
- metric >= 0 AND
- metrics_type = '{mae}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out))
-FROM evaluate_out;
-
--- Test that passing NULL / None instead of 0 for gpus_per_host works
-DROP TABLE IF EXISTS evaluate_out;
-SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out');
-SELECT assert(loss >= 0 AND
- metric >= 0 AND
- metrics_type = '{mae}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out))
-FROM evaluate_out;
-
--- Test that evaluate errors out correctly if mst_key is given for non-multi model tables
-DROP TABLE IF EXISTS evaluate_out;
-SELECT assert(trap_error($TRAP$
- SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out', FALSE ,1);
- $TRAP$) = 1, 'Should error out if mst_key is given for non-multi model tables');
-
--- Test that evaluate errors out correctly if model_arch field missing from fit output
-DROP TABLE IF EXISTS evaluate_out;
-ALTER TABLE keras_saved_out DROP COLUMN model_arch;
-SELECT assert(trap_error($TRAP$
- SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out');
- $TRAP$) = 1, 'Should error out if model_arch column is missing from model_table');
-
-
-m4_changequote(`<!', `!>')
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
-
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$,
- $$batch_size=10,epochs=1$$
- ]
-);
-
-DROP TABLE if exists cifar_10_multiple_model, cifar_10_multiple_model_summary,
- cifar_10_multiple_model_info;
-SELECT setseed(0);
-SELECT madlib_keras_fit_multiple_model(
- 'cifar_10_sample_batched',
- 'cifar_10_multiple_model',
- 'mst_table',
- 6,
- FALSE
-);
-
-DROP TABLE IF EXISTS evaluate_out;
-SELECT madlib_keras_evaluate('cifar_10_multiple_model', 'cifar_10_sample_batched', 'evaluate_out', FALSE, 2);
-SELECT assert(relative_error(e.metric,i.training_metrics_final) < 0.00001 AND
- relative_error(e.loss,i.training_loss_final) < 0.00001 AND
- e.metrics_type = '{accuracy}', 'Evaluate output validation failed.')
-FROM evaluate_out e, cifar_10_multiple_model_info i WHERE i.mst_key = 2;
-
-
--- Test that evaluate errors out correctly if mst_key is missing for multi model tables
-DROP TABLE IF EXISTS evaluate_out;
-SELECT assert(trap_error($TRAP$
- SELECT madlib_keras_evaluate('cifar_10_multiple_model', 'cifar_10_sample_val', 'evaluate_out');
- $TRAP$) = 1, 'Should error out if mst_key is missing for multi model tables');
-
-
-!>)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
deleted file mode 100644
index a35eb6b..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
+++ /dev/null
@@ -1,419 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_cifar.setup.sql_in'
-)
-
-m4_include(`SQLCommon.m4')
-
--- Please do not break up the compile_params string
--- It might break the assertion
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_saved_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3,
- NULL,
- 'cifar_10_sample_val');
-
-SELECT assert(
- model_arch_table = 'model_arch' AND
- model_id = 1 AND
- model_type = 'madlib_keras' AND
- start_training_time < now() AND
- end_training_time > start_training_time AND
- source_table = 'cifar_10_sample_batched' AND
- validation_table = 'cifar_10_sample_val' AND
- model = 'keras_saved_out' AND
- dependent_varname = 'y' AND
- dependent_vartype = 'smallint' AND
- independent_varname = 'x' AND
- normalizing_const = 255.0 AND
- pg_typeof(normalizing_const) = 'real'::regtype AND
- name is NULL AND
- description is NULL AND
- model_size > 0 AND
- madlib_version is NOT NULL AND
- compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text AND
- fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
- num_iterations = 3 AND
- metrics_compute_frequency = 3 AND
- num_classes = 2 AND
- class_values = '{0,1}' AND
- metrics_type = '{mae}' AND
- training_metrics_final >= 0 AND
- training_loss_final >= 0 AND
- array_upper(training_metrics, 1) = 1 AND
- array_upper(training_loss, 1) = 1 AND
- array_upper(metrics_elapsed_time, 1) = 1 AND
- validation_metrics_final >= 0 AND
- validation_loss_final >= 0 AND
- array_upper(validation_metrics, 1) = 1 AND
- array_upper(validation_loss, 1) = 1 ,
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM keras_saved_out_summary) summary;
-
-SELECT assert(
- model_weights IS NOT NULL AND
- model_arch IS NOT NULL, 'Keras model output validation failed. Actual:' || __to_char(k))
-FROM (SELECT * FROM keras_saved_out) k;
-
--- Verify number of iterations for which metrics and loss are computed
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_saved_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3,
- NULL,
- 'cifar_10_sample_val',
- 2);
-SELECT assert(
- num_iterations = 3 AND
- metrics_compute_frequency = 2 AND
- training_metrics_final >= 0 AND
- training_loss_final >= 0 AND
- metrics_type = '{accuracy}' AND
- array_upper(training_metrics, 1) = 2 AND
- array_upper(training_loss, 1) = 2 AND
- array_upper(metrics_elapsed_time, 1) = 2 AND
- validation_metrics_final >= 0 AND
- validation_loss_final >= 0 AND
- array_upper(validation_metrics, 1) = 2 AND
- array_upper(validation_loss, 1) = 2 ,
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM keras_saved_out_summary) summary;
-
--- Test for
- -- Non null name and description columns
- -- Null validation table
-DROP TABLE IF EXISTS keras_out, keras_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 2,
- NULL,
- NULL,
- 1,
- NULL,
- 'model name',
- 'model desc');
-
-SELECT assert(
- source_table = 'cifar_10_sample_batched' AND
- model = 'keras_out' AND
- dependent_varname = 'y' AND
- independent_varname = 'x' AND
- model_arch_table = 'model_arch' AND
- model_id = 1 AND
- compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND
- fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
- num_iterations = 2 AND
- validation_table is NULL AND
- metrics_compute_frequency = 1 AND
- name = 'model name' AND
- description = 'model desc' AND
- model_type = 'madlib_keras' AND
- model_size > 0 AND
- start_training_time < now() AND
- end_training_time > start_training_time AND
- array_upper(metrics_elapsed_time, 1) = 2 AND
- dependent_vartype = 'smallint' AND
- madlib_version is NOT NULL AND
- num_classes = 2 AND
- class_values = '{0,1}' AND
- metrics_type = '{accuracy}' AND
- normalizing_const = 255.0 AND
- training_metrics_final is not NULL AND
- training_loss_final is not NULL AND
- array_upper(training_metrics, 1) = 2 AND
- array_upper(training_loss, 1) = 2 AND
- validation_metrics_final is NULL AND
- validation_loss_final is NULL AND
- validation_metrics is NULL AND
- validation_loss is NULL,
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM keras_out_summary) summary;
-
-SELECT assert(model_weights IS NOT NULL , 'Keras model output validation failed') FROM (SELECT * FROM keras_out) k;
-
--- Validate metrics=NULL works with fit
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
-'cifar_10_sample_batched',
-'keras_saved_out',
-'model_arch',
-1,
-$$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy'$$::text,
-$$ batch_size=2, epochs=1, verbose=0 $$::text,
-1);
-
-SELECT assert(
- metrics_type is NULL AND
- training_metrics IS NULL AND
- array_upper(training_loss, 1) = 1 AND
- array_upper(metrics_elapsed_time, 1) = 1 AND
- validation_metrics_final IS NULL AND
- validation_loss_final >= 0 AND
- validation_metrics IS NULL AND
- array_upper(validation_loss, 1) = 1,
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM keras_saved_out_summary) summary;
-
--- Validate metrics=[] works with fit
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
-'cifar_10_sample_batched',
-'keras_saved_out',
-'model_arch',
-1,
-$$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=[]$$::text,
-$$ batch_size=2, epochs=1, verbose=0 $$::text,
-1);
-
-SELECT assert(
- metrics_type IS NULL AND
- training_metrics IS NULL AND
- array_upper(training_loss, 1) = 1 AND
- array_upper(metrics_elapsed_time, 1) = 1 AND
- validation_metrics_final IS NULL AND
- validation_loss_final >= 0 AND
- validation_metrics IS NULL AND
- array_upper(validation_loss, 1) = 1,
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM keras_saved_out_summary) summary;
-
--- Compile and fit parameter tests
-DROP TABLE IF EXISTS keras_out, keras_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_out',
- 'model_arch',
- 1,
- $$ optimizer='SGD', loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 1,
- NULL,
- NULL,
- NULL,
- NULL, 'model name', 'model desc');
-
-DROP TABLE IF EXISTS keras_out, keras_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_out',
- 'model_arch',
- 1,
- $$ optimizer='Adam()', loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 1,
- NULL,
- NULL,
- NULL,
- NULL, 'model name', 'model desc');
-
-DROP TABLE IF EXISTS keras_out, keras_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_out',
- 'model_arch',
- 1,
- $$ optimizer=Adam(epsilon=None), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 1,
- FALSE,
- NULL,
- NULL,
- NULL, 'model name', 'model desc');
-
-DROP TABLE IF EXISTS keras_out, keras_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), metrics=['accuracy'], loss_weights=[2], sample_weight_mode=None, loss='categorical_crossentropy' $$::text,
- $$ epochs=10, verbose=0, shuffle=True, initial_epoch=1, steps_per_epoch=2 $$::text,
- 1,
- NULL,
- NULL,
- NULL,
- False, 'model name', 'model desc');
-
-DROP TABLE IF EXISTS keras_out, keras_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), metrics=['accuracy'], loss_weights=[2], sample_weight_mode=None, loss='categorical_crossentropy' $$::text,
- NULL,
- 1,
- NULL,
- NULL,
- NULL,
- False, 'model name', 'model desc');
-
--- -- negative test case for passing non numeric y to fit
--- induce failure by passing a non numeric column
-DROP TABLE IF EXISTS cifar_10_sample_val_failure;
-CREATE TABLE cifar_10_sample_val_failure AS SELECT * FROM cifar_10_sample_val;
-ALTER TABLE cifar_10_sample_val_failure rename dependent_var to dependent_var_original;
-ALTER TABLE cifar_10_sample_val_failure rename buffer_id to dependent_var;
-DROP TABLE IF EXISTS keras_out, keras_out_summary;
-SELECT assert(trap_error($TRAP$SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 2,
- NULL,
- 'cifar_10_sample_val_failure');$TRAP$) = 1,
- 'Passing y of type non numeric array to fit should error out.');
-
--- Tests with text class values:
--- Modify input data to have text classes, and mini-batch it.
--- Create a new table using the text based column for dep var.
-DROP TABLE IF EXISTS cifar_10_sample_text_batched;
-m4_changequote(`<!', `!>')
-CREATE TABLE cifar_10_sample_text_batched AS
- SELECT buffer_id, independent_var, dependent_var,
- independent_var_shape, dependent_var_shape
- m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!, __dist_key__ !>)
- FROM cifar_10_sample_batched m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! DISTRIBUTED BY (__dist_key__) !>);
-
--- Insert a new row with NULL as the dependent var (one-hot encoded)
-UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0;
-UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1;
-INSERT INTO cifar_10_sample_text_batched(m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) buffer_id, independent_var, dependent_var, independent_var_shape, dependent_var_shape)
- SELECT m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) 2 AS buffer_id, independent_var,
- convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS dependent_var,
- independent_var_shape, dependent_var_shape
- FROM cifar_10_sample_batched WHERE cifar_10_sample_batched.buffer_id=0;
-UPDATE cifar_10_sample_text_batched SET dependent_var_shape = ARRAY[1,5];
-
--- Create the necessary summary table for the batched input.
-DROP TABLE IF EXISTS cifar_10_sample_text_batched_summary;
-CREATE TABLE cifar_10_sample_text_batched_summary(
- source_table text,
- output_table text,
- dependent_varname text,
- independent_varname text,
- dependent_vartype text,
- class_values text[],
- buffer_size integer,
- normalizing_const numeric);
-INSERT INTO cifar_10_sample_text_batched_summary values (
- 'cifar_10_sample',
- 'cifar_10_sample_text_batched',
- 'y_text',
- 'x',
- 'text',
- ARRAY[NULL,'cat','dog',NULL,NULL],
- 1,
- 255.0);
-
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_text_batched',
- 'keras_saved_out',
- 'model_arch',
- 2,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
--- Assert fit has correct class_values
-SELECT assert(
- dependent_vartype = 'text' AND
- class_values = '{NULL,cat,dog,NULL,NULL}',
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM keras_saved_out_summary) summary;
-
--- Test with INTEGER class_values
--- with NULL as a valid class value
-INSERT INTO cifar_10_sample(id, x, y, imgpath)
-SELECT 3, x, NULL, '0/img3.jpg' FROM cifar_10_sample
-WHERE y = 1;
-INSERT INTO cifar_10_sample(id, x, y, imgpath)
-SELECT 4, x, 4, '0/img4.jpg' FROM cifar_10_sample
-WHERE y = 0;
-INSERT INTO cifar_10_sample(id, x, y, imgpath)
-SELECT 5, x, 5, '0/img5.jpg' FROM cifar_10_sample
-WHERE y = 1;
-
-DROP TABLE IF EXISTS cifar_10_sample_int_batched;
-DROP TABLE IF EXISTS cifar_10_sample_int_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, 5);
-
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_int_batched',
- 'keras_saved_out',
- 'model_arch',
- 2,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
-
--- Assert fit has correct class_values
-SELECT assert(
- dependent_vartype = 'smallint' AND
- class_values = '{NULL,0,1,4,5}',
- 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM keras_saved_out_summary) summary;
-
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_test_shape_batched',
- 'keras_saved_out',
- 'model_arch',
- 3,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
-
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-CREATE TABLE "special-char?" AS SELECT * FROM model_arch;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_test_shape_batched',
- 'keras_saved_out',
- '"special-char?"',
- 3,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_gpu_info.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_gpu_info.sql_in
deleted file mode 100644
index a8376db..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_gpu_info.sql_in
+++ /dev/null
@@ -1,42 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-SELECT assert(trap_error($$SELECT * from gpu_configuration('output_table', 'invalid_source')$$) = 1, 'Invalid source');
-
-DROP TABLE IF EXISTS output_table_tf;
-SELECT gpu_configuration('output_table_tf');
-SELECT * FROM output_table_tf;
-
-DROP TABLE IF EXISTS output_table_tf;
-SELECT gpu_configuration('output_table_tf', NULL);
-SELECT * FROM output_table_tf;
-
-DROP TABLE IF EXISTS output_table_nvidia;
-SELECT gpu_configuration('output_table_nvidia','NVIDIA');
-SELECT * FROM output_table_nvidia;
-
-DROP TABLE IF EXISTS output_table_tf;
-SELECT gpu_configuration('output_table_tf', 'tensorflow');
-SELECT * FROM output_table_tf;
-
-DROP TABLE IF EXISTS output_table_tf;
-SELECT * FROM gpu_configuration('output_table_tf');
-SELECT * FROM output_table_tf;
\ No newline at end of file
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in
deleted file mode 100644
index 389efa3..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in
+++ /dev/null
@@ -1,295 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
--------------------- IRIS test input tables -----------------
-
-DROP TABLE IF EXISTS iris_data;
-CREATE TABLE iris_data(
- id serial,
- attributes numeric[],
- class_text varchar
-);
-INSERT INTO iris_data(id, attributes, class_text) VALUES
-(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'),
-(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'),
-(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'),
-(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'),
-(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'),
-(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'),
-(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'),
-(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'),
-(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'),
-(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'),
-(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'),
-(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'),
-(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'),
-(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'),
-(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'),
-(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'),
-(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'),
-(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'),
-(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'),
-(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'),
-(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'),
-(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'),
-(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'),
-(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'),
-(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'),
-(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'),
-(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'),
-(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'),
-(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'),
-(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'),
-(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'),
-(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'),
-(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'),
-(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'),
-(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'),
-(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'),
-(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'),
-(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'),
-(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'),
-(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'),
-(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'),
-(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'),
-(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'),
-(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'),
-(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'),
-(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'),
-(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'),
-(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'),
-(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'),
-(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'),
-(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'),
-(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'),
-(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'),
-(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'),
-(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'),
-(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'),
-(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'),
-(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'),
-(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'),
-(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'),
-(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'),
-(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'),
-(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'),
-(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'),
-(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'),
-(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'),
-(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'),
-(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'),
-(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'),
-(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'),
-(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'),
-(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'),
-(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'),
-(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'),
-(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'),
-(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'),
-(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'),
-(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'),
-(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'),
-(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'),
-(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'),
-(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'),
-(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'),
-(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'),
-(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'),
-(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'),
-(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'),
-(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'),
-(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'),
-(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'),
-(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'),
-(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'),
-(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'),
-(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'),
-(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'),
-(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'),
-(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'),
-(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'),
-(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'),
-(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'),
-(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'),
-(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'),
-(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'),
-(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'),
-(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'),
-(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'),
-(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'),
-(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'),
-(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'),
-(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'),
-(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'),
-(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'),
-(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'),
-(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'),
-(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'),
-(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'),
-(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'),
-(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'),
-(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'),
-(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'),
-(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'),
-(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'),
-(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'),
-(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'),
-(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'),
-(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'),
-(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'),
-(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'),
-(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'),
-(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'),
-(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'),
-(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'),
-(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'),
-(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'),
-(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'),
-(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'),
-(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'),
-(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'),
-(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'),
-(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'),
-(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'),
-(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'),
-(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'),
-(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'),
-(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica');
-
-
-DROP TABLE IF EXISTS iris_data_one_hot_encoded;
-CREATE TABLE iris_data_one_hot_encoded as select id, attributes, ARRAY[class_text is not distinct from 'Iris-setosa', class_text is not distinct from 'Iris-versicolor', class_text is not distinct from 'Iris-virginica']::int[] as class_one_hot_encoded
-from iris_data;
-
-
-DROP TABLE IF EXISTS iris_data_packed, iris_data_packed_summary;
-SELECT training_preprocessor_dl('iris_data', -- Source table
- 'iris_data_packed', -- Output table
- 'class_text', -- Dependent variable
- 'attributes' -- Independent variable
- );
-
-DROP TABLE IF EXISTS iris_data_one_hot_encoded_packed, iris_data_one_hot_encoded_packed_summary;
-SELECT training_preprocessor_dl('iris_data_one_hot_encoded', -- Source table
- 'iris_data_one_hot_encoded_packed', -- Output table
- 'class_one_hot_encoded', -- Dependent variable
- 'attributes' -- Independent variable
- );
-
-DROP TABLE IF EXISTS iris_data_val, iris_data_val_summary;
-SELECT validation_preprocessor_dl('iris_data', -- Source table
- 'iris_data_val', -- Output table
- 'class_text', -- Dependent variable
- 'attributes', -- Independent variable
- 'iris_data_packed'-- Training preprocessed table
- );
-
-DROP TABLE IF EXISTS iris_data_one_hot_encoded_val, iris_data_one_hot_encoded_val_summary;
-SELECT validation_preprocessor_dl('iris_data_one_hot_encoded', -- Source table
- 'iris_data_one_hot_encoded_val', -- Output table
- 'class_one_hot_encoded', -- Dependent variable
- 'attributes', -- Independent variable
- 'iris_data_one_hot_encoded_packed' -- Training preprocessed table
- );
-
-DROP TABLE IF EXISTS iris_model_arch;
--- NOTE: The seed is set to 0 for every layer.
-SELECT load_keras_model('iris_model_arch', -- Output table,
-$$
-{
-"class_name": "Sequential",
-"keras_version": "2.1.6",
-"config":
- [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling",
- "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}},
- "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null,
- "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true,
- "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
- "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true,
- "activity_regularizer": null}}, {"class_name": "Dense",
- "config": {"kernel_initializer": {"class_name": "VarianceScaling",
- "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}},
- "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null,
- "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null,
- "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true,
- "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer":
- {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0,
- "seed": 0, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null,
- "bias_regularizer": null, "bias_constraint": null, "activation": "softmax",
- "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
- "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}],
- "backend": "tensorflow"}
-$$
-);
-
--- Duplicate the architecture, but note that trainable is set to FALSE.
--- This is to ensure we don't learn anything new, that would help us
--- deterministically assert the accuracy and loss after transfer learning
--- and warm start.
-SELECT load_keras_model('iris_model_arch', -- Output table,
-$$
-{
-"class_name": "Sequential",
-"keras_version": "2.1.6",
-"config":
- [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling",
- "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}},
- "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null,
- "bias_constraint": null, "dtype": "float32", "activation": "relu",
- "trainable": false,
- "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
- "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true,
- "activity_regularizer": null}}, {"class_name": "Dense",
- "config": {"kernel_initializer": {"class_name": "VarianceScaling",
- "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}},
- "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null,
- "bias_constraint": null, "activation": "relu",
- "trainable": false,
- "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
- "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}},
- {"class_name": "Dense", "config": {"kernel_initializer":
- {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0,
- "seed": 0, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null,
- "bias_regularizer": null, "bias_constraint": null, "activation": "softmax",
- "trainable": false,
- "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
- "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}],
- "backend": "tensorflow"}
-$$
-);
-
-DROP TABLE IF EXISTS iris_train, iris_test;
--- Set seed so results are reproducible
-SELECT setseed(0);
-SELECT train_test_split('iris_data', -- Source table
- 'iris', -- Output table root name
- 0.8, -- Train proportion
- NULL, -- Test proportion (0.2)
- NULL, -- Strata definition
- NULL, -- Output all columns
- NULL, -- Sample without replacement
- TRUE -- Separate output tables
- );
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in
deleted file mode 100644
index 20e2332..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in
+++ /dev/null
@@ -1,139 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-m4_include(`SQLCommon.m4')
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
-)
-
-DROP TABLE if exists pg_temp.iris_model, pg_temp.iris_model_summary;
-SELECT madlib_keras_fit(
- 'iris_data_packed',
- 'pg_temp.iris_model',
- 'iris_model_arch',
- 1,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$,
- $$batch_size=16, epochs=1$$,
- 3,
- FALSE
-);
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END;
-
-SELECT assert(
- model_arch_table = 'iris_model_arch' AND
- validation_table is NULL AND
- source_table = 'iris_data_packed' AND
- model = 'pg_temp.iris_model' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
- madlib_version is NOT NULL AND
- num_iterations = 3 AND
- start_training_time < now() AND
- end_training_time < now() AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype LIKE '%char%' AND
- normalizing_const = 1,
- 'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM pg_temp.iris_model_summary) summary;
-
--- Run Predict
-DROP TABLE IF EXISTS pg_temp.iris_predict;
-SELECT madlib_keras_predict(
- 'pg_temp.iris_model',
- 'iris_data',
- 'id',
- 'attributes',
- 'pg_temp.iris_predict',
- 'prob',
- FALSE);
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END;
-
--- Run Evaluate
-DROP TABLE IF EXISTS pg_temp.evaluate_out;
-SELECT madlib_keras_evaluate(
- 'pg_temp.iris_model',
- 'iris_data_val',
- 'pg_temp.evaluate_out',
- FALSE);
-
-SELECT assert(loss >= 0 AND
- metric >= 0 AND
- metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out))
-FROM pg_temp.evaluate_out;
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END;
-
--- Test for one-hot encoded user input data
-DROP TABLE if exists iris_model, iris_model_summary, iris_model_info;
-SELECT madlib_keras_fit(
- 'iris_data_one_hot_encoded_packed',
- 'iris_model',
- 'iris_model_arch',
- 1,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$,
- $$batch_size=16, epochs=1$$,
- 3,
- FALSE
-);
-
-SELECT assert(
- model_arch_table = 'iris_model_arch' AND
- validation_table is NULL AND
- source_table = 'iris_data_one_hot_encoded_packed' AND
- model = 'iris_model' AND
- dependent_varname = 'class_one_hot_encoded' AND
- independent_varname = 'attributes' AND
- madlib_version is NOT NULL AND
- num_iterations = 3 AND
- start_training_time < now() AND
- end_training_time < now() AND
- dependent_vartype = 'integer[]' AND
- num_classes = NULL AND
- class_values = NULL AND
- normalizing_const = 1,
- 'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary))
-FROM (SELECT * FROM iris_model_summary) summary;
-
--- Run Predict
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib_keras_predict(
- 'iris_model',
- 'iris_data_one_hot_encoded',
- 'id',
- 'attributes',
- 'iris_predict',
- 'prob',
- FALSE);
-
--- Run Evaluate
-DROP TABLE IF EXISTS evaluate_out;
-SELECT madlib_keras_evaluate(
- 'iris_model',
- 'iris_data_one_hot_encoded_val',
- 'evaluate_out',
- FALSE);
-
-SELECT assert(loss >= 0 AND
- metric >= 0 AND
- metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out))
-FROM evaluate_out;
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
deleted file mode 100644
index e1dbe0c..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
+++ /dev/null
@@ -1,426 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-m4_include(`SQLCommon.m4')
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
-)
-
--- MST table generation tests
--- Valid inputs should pass and yield 6 msts in the table
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$,
- $$batch_size=10,epochs=1$$
- ]
-);
-SELECT assert(
- COUNT(*)=6,
- 'The length of mst table does not match with the inputs'
-)
-FROM mst_table;
-
--- Test summary table output
-SELECT assert(
- model_arch_table = 'iris_model_arch',
- 'Model selection output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM mst_table_summary) summary;
-
--- Invalid arguments must be errored out
-
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT assert(trap_error($TRAP$
- SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[-1],
- ARRAY[
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$
- ]
- );
-$TRAP$)=1, 'Should error out if model_id is not in the model arch table');
-
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT assert(trap_error($TRAP$
- SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$foo='bar'$$
- ],
- ARRAY[
- $$batch_size='bar'$$
- ]
- );
-$TRAP$)=1, 'Should error out if the provided parameters are not valid');
-
--- Must deduplicate, options with extra white spaces should not be considered
--- as distinct params.
-
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
- $$ loss='categorical_crossentropy', optimizer='Adam(lr=0.1)',metrics=['accuracy'] $$,
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$,
- $$batch_size=10,epochs=1$$
- ]
-);
-SELECT assert(
- COUNT(*)=4,
- 'The length of mst table (' || COUNT(*) || ')does not match with the inputs due to deduplication failure'
-)
-FROM mst_table;
-
--- Must also handle duplicates where order of key/value pairs is re-arranged
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
- $$metrics= ['accuracy'], loss='categorical_crossentropy', optimizer='Adam(lr=0.1)'$$,
- $$loss='mse',optimizer='Adam(lr=0.001)', metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$,
- $$epochs=1, batch_size=5$$
- ]
-);
-SELECT assert(
- COUNT(*)=2,
- 'The length of mst table (' || COUNT(*) || ') does not match with the inputs due to deduplication failure'
-)
-FROM mst_table;
-
-m4_changequote(`<!', `!>')
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
--- Multiple models test
--- Prepare model selection table with three rows
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)', metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.0001)', metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=50, epochs=1$$
- ]
-);
--- Prepare model selection table with only one row
-DROP TABLE IF EXISTS mst_table_1row, mst_table_1row_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table_1row',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=16, epochs=1$$
- ]
-);
--- Prepare model selection table with four rows
-DROP TABLE IF EXISTS mst_table_4row, mst_table_4row_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table_4row',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)', metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=16, epochs=1$$,
- $$batch_size=32, epochs=1$$
- ]
-);
-
--- Test for one-hot encoded input data
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_one_hot_encoded_packed',
- 'iris_multiple_model',
- 'mst_table_4row',
- 3,
- FALSE
-);
-
-SELECT assert(
- model_arch_table = 'iris_model_arch' AND
- validation_table is NULL AND
- model_info = 'iris_multiple_model_info' AND
- source_table = 'iris_data_one_hot_encoded_packed' AND
- model = 'iris_multiple_model' AND
- dependent_varname = 'class_one_hot_encoded' AND
- independent_varname = 'attributes' AND
- madlib_version is NOT NULL AND
- num_iterations = 3 AND
- start_training_time < now() AND
- end_training_time < now() AND
- dependent_vartype = 'integer[]' AND
- num_classes = NULL AND
- class_values = NULL AND
- normalizing_const = 1 AND
- metrics_iters = ARRAY[3],
- 'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary))
-FROM (SELECT * FROM iris_multiple_model_summary) summary;
-
--- Test the output table created are all persistent(not unlogged)
-SELECT assert(MADLIB_SCHEMA.is_table_unlogged('iris_multiple_model') = false, 'Model output table is unlogged');
-SELECT assert(MADLIB_SCHEMA.is_table_unlogged('iris_multiple_model_summary') = false, 'Model summary output table is unlogged');
-SELECT assert(MADLIB_SCHEMA.is_table_unlogged('iris_multiple_model_info') = false, 'Model info output table is unlogged');
-
--- Test when number of configs(3) equals number of segments(3)
-DROP TABLE IF EXISTS iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT setseed(0);
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 6,
- FALSE,
- 'iris_data_one_hot_encoded_packed'
-);
-
-SELECT assert(
- source_table = 'iris_data_packed' AND
- validation_table = 'iris_data_one_hot_encoded_packed' AND
- model = 'iris_multiple_model' AND
- model_info = 'iris_multiple_model_info' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
- model_arch_table = 'iris_model_arch' AND
- num_iterations = 6 AND
- start_training_time < now() AND
- end_training_time < now() AND
- madlib_version is NOT NULL AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype LIKE '%char%' AND
- normalizing_const = 1 AND
- name IS NULL AND
- description IS NULL AND
- metrics_compute_frequency = 6,
- 'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM iris_multiple_model_summary) summary;
-
-SELECT assert(COUNT(*)=3, 'Info table must have exactly same rows as the number of msts.')
-FROM iris_multiple_model_info;
-
-SELECT assert(
- model_id = 1 AND
- model_type = 'madlib_keras' AND
- model_size > 0 AND
- fit_params = $MAD$batch_size=50, epochs=1$MAD$::text AND
- metrics_type = '{accuracy}' AND
- training_metrics_final >= 0 AND
- training_loss_final >= 0 AND
- array_upper(training_metrics, 1) = 1 AND
- array_upper(training_loss, 1) = 1 AND
- validation_metrics_final >= 0 AND
- validation_loss_final >= 0 AND
- array_upper(validation_metrics, 1) = 1 AND
- array_upper(validation_loss, 1) = 1 AND
- array_upper(metrics_elapsed_time, 1) = 1,
- 'Keras Fit Multiple Output Info Validation failed. Actual:' || __to_char(info))
-FROM (SELECT * FROM iris_multiple_model_info) info;
-
-SELECT assert(cnt = 1,
- 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info))
-FROM (SELECT count(*) cnt FROM iris_multiple_model_info
-WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$MAD$::text) info;
-
-SELECT assert(cnt = 1,
- 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info))
-FROM (SELECT count(*) cnt FROM iris_multiple_model_info
-WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)', metrics=['accuracy']$MAD$::text) info;
-
-SELECT assert(cnt = 1,
- 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info))
-FROM (SELECT count(*) cnt FROM iris_multiple_model_info
-WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.0001)', metrics=['accuracy']$MAD$::text) info;
-
-SELECT assert(
- training_loss[6]-training_loss[1] < 0.1 AND
- training_metrics[6]-training_metrics[1] > -0.1,
- 'The loss and accuracy should have improved with more iterations.'
-)
-FROM iris_multiple_model_info
-WHERE compile_params like '%lr=0.001%';
-
--- Test when number of configs(1) is less than number of segments(3)
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table_1row',
- 3,
- FALSE,
- NULL,
- 1,
- FALSE,
- 'multi_model_name',
- 'multi_model_descr'
-);
-
-SELECT assert(COUNT(*)=1, 'Info table must have exactly same rows as the number of msts.')
-FROM iris_multiple_model_info;
-
-SELECT assert(
- model_id = 1 AND
- model_type = 'madlib_keras' AND
- model_size > 0 AND
- fit_params = $MAD$batch_size=16, epochs=1$MAD$::text AND
- metrics_type = '{accuracy}' AND
- training_metrics_final >= 0 AND
- training_loss_final >= 0 AND
- array_upper(training_metrics, 1) = 3 AND
- array_upper(training_loss, 1) = 3 AND
- array_upper(metrics_elapsed_time, 1) = 3,
- 'Keras Fit Multiple Output Info Validation failed. Actual:' || __to_char(info))
-FROM (SELECT * FROM iris_multiple_model_info) info;
-
-SELECT assert(metrics_elapsed_time[3] - metrics_elapsed_time[1] > 0,
- 'Keras Fit Multiple invalid elapsed time calculation.')
-FROM (SELECT * FROM iris_multiple_model_info) info;
-
-SELECT assert(
- name = 'multi_model_name' AND
- description = 'multi_model_descr' AND
- metrics_compute_frequency = 1,
- 'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM iris_multiple_model_summary) summary;
-
-SELECT assert(cnt = 1,
- 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info))
-FROM (SELECT count(*) cnt FROM iris_multiple_model_info
-WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$MAD$::text) info;
-
--- Test when number of configs(4) larger than number of segments(3)
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table_4row',
- 3,
- FALSE
-);
-
--- The default value of the guc 'dev_opt_unsafe_truncate_in_subtransaction' is 'off'
--- but we change it to 'on' in fit_multiple.py. Assert that the value is
--- reset after calling fit_multiple
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('dev_opt_unsafe_truncate_in_subtransaction', 'off') END;
-
-SELECT assert(COUNT(*)=4, 'Info table must have exactly same rows as the number of msts.')
-FROM iris_multiple_model_info;
-
-SELECT assert(
- model_id = 1 AND
- model_type = 'madlib_keras' AND
- model_size > 0 AND
- metrics_type = '{accuracy}' AND
- training_metrics_final >= 0 AND
- training_loss_final >= 0 AND
- array_upper(training_metrics, 1) = 1 AND
- array_upper(training_loss, 1) = 1 AND
- array_upper(metrics_elapsed_time, 1) = 1,
- 'Keras Fit Multiple Output Info Validation failed. Actual:' || __to_char(info))
-FROM (SELECT * FROM iris_multiple_model_info) info;
-
-SELECT assert(cnt = 1,
- 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info))
-FROM (SELECT count(*) cnt FROM iris_multiple_model_info
-WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$MAD$::text
-AND fit_params = $MAD$batch_size=32, epochs=1$MAD$::text) info;
-
--- Test when class values have NULL values
-UPDATE iris_data_packed_summary SET class_values = ARRAY['Iris-setosa','Iris-versicolor',NULL];
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table_1row',
- 1,
- FALSE,
- NULL,
- 1,
- FALSE
-);
-
-SELECT assert(
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,NULL}',
- 'Keras Fit Multiple num_clases and class values Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM iris_multiple_model_summary) summary;
-
-------------- Test for schema qualified output table and input table -----------------------
-CREATE SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__;
-CREATE TABLE __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_data_packed as select * from iris_data_packed;
-CREATE TABLE __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_data_packed_summary as select * from iris_data_packed_summary;
-
--- do not drop the output table created in the previous test
-SELECT count(*) from iris_multiple_model;
-SELECT madlib_keras_fit_multiple_model(
- '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_data_packed',
- '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model',
- 'mst_table_1row',
- 1,
- FALSE,
- NULL,
- 1,
- FALSE
-);
-SELECT count(*) from __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model;
-SELECT assert(
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,NULL}',
- 'Keras Fit Multiple validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_summary) summary;
-
-DROP SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__ CASCADE;
-!>)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in
deleted file mode 100644
index d738f48..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in
+++ /dev/null
@@ -1,165 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-m4_include(`SQLCommon.m4')
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
-)
-
-m4_changequote(`<!', `!>')
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
--- Multiple models End-to-End test
--- Prepare model selection table with four rows
-DROP TABLE IF EXISTS pg_temp.mst_table, pg_temp.mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'pg_temp.mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)', metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.0001)', metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=16, epochs=1$$
- ]
-);
-
-CREATE SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__;
-DROP TABLE if exists __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model,
- __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_summary,
- __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_info;
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model',
- 'pg_temp.mst_table',
- 3,
- FALSE
-);
-
-SELECT assert(
- model_arch_table = 'iris_model_arch' AND
- validation_table is NULL AND
- model_info = '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_info' AND
- source_table = 'iris_data_packed' AND
- model = '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model' AND
- dependent_varname = 'class_text' AND
- independent_varname = 'attributes' AND
- madlib_version is NOT NULL AND
- num_iterations = 3 AND
- start_training_time < now() AND
- end_training_time < now() AND
- num_classes = 3 AND
- class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
- dependent_vartype LIKE '%char%' AND
- normalizing_const = 1,
- 'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary))
-FROM (SELECT * FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_summary) summary;
-
--- Run Predict
-DROP TABLE IF EXISTS pg_temp.iris_predict;
-SELECT madlib_keras_predict(
- '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model',
- 'iris_data',
- 'id',
- 'attributes',
- 'pg_temp.iris_predict',
- 'prob',
- NULL,
- 1);
-
--- Run Evaluate
-DROP TABLE IF EXISTS pg_temp.evaluate_out;
-SELECT madlib_keras_evaluate(
- '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model',
- 'iris_data_val',
- 'pg_temp.evaluate_out',
- NULL,
- 1);
-
-SELECT assert(loss >= 0 AND
- metric >= 0 AND
- metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out))
-FROM pg_temp.evaluate_out;
-
--- Test for one-hot encoded user input data
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_one_hot_encoded_packed',
- 'iris_multiple_model',
- 'pg_temp.mst_table',
- 3,
- FALSE
-);
-
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END;
-
-SELECT assert(
- model_arch_table = 'iris_model_arch' AND
- validation_table is NULL AND
- model_info = 'iris_multiple_model_info' AND
- source_table = 'iris_data_one_hot_encoded_packed' AND
- model = 'iris_multiple_model' AND
- dependent_varname = 'class_one_hot_encoded' AND
- independent_varname = 'attributes' AND
- madlib_version is NOT NULL AND
- num_iterations = 3 AND
- start_training_time < now() AND
- end_training_time < now() AND
- dependent_vartype = 'integer[]' AND
- num_classes = NULL AND
- class_values = NULL AND
- normalizing_const = 1,
- 'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary))
-FROM (SELECT * FROM iris_multiple_model_summary) summary;
-
--- Run Predict
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib_keras_predict(
- 'iris_multiple_model',
- 'iris_data_one_hot_encoded',
- 'id',
- 'attributes',
- 'iris_predict',
- 'prob',
- NULL,
- 1);
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END;
-
--- Run Evaluate
-DROP TABLE IF EXISTS evaluate_out;
-SELECT madlib_keras_evaluate(
- 'iris_multiple_model',
- 'iris_data_one_hot_encoded_val',
- 'evaluate_out',
- NULL,
- 1);
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END;
-
-SELECT assert(loss >= 0 AND
- metric >= 0 AND
- metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out))
-FROM evaluate_out;
-
-DROP SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__ CASCADE;
-!>)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
deleted file mode 100644
index 5a5b362..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
+++ /dev/null
@@ -1,403 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-m4_include(`SQLCommon.m4')
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_cifar.setup.sql_in'
-)
-
--- Please do not break up the compile_params string
--- It might break the assertion
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_batched',
- 'keras_saved_out',
- 'model_arch',
- 1,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
-
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample',
- 'id',
- 'x',
- 'cifar10_predict',
- NULL,
- FALSE);
-
--- Validate that prediction output table exists and has correct schema
-SELECT assert(UPPER(pg_typeof(id)::TEXT) = 'INTEGER', 'id column should be INTEGER type')
- FROM cifar10_predict;
-
-SELECT assert(UPPER(pg_typeof(estimated_y)::TEXT) =
- 'SMALLINT', 'prediction column should be SMALLINT type')
- FROM cifar10_predict;
-
--- Validate correct number of rows returned.
-SELECT assert(COUNT(*)=2, 'Output table of madlib_keras_predict should have two rows')
-FROM cifar10_predict;
-
--- First test that all values are in set of class values; if this breaks, it's definitely a problem.
-SELECT assert(estimated_y IN (0,1),
- 'Predicted value not in set of defined class values for model')
-FROM cifar10_predict;
-
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT assert(trap_error($TRAP$SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample_batched',
- 'buffer_id',
- 'independent_var',
- 'cifar10_predict',
- NULL,
- FALSE);$TRAP$) = 1,
- 'Passing batched image table to predict should error out.');
-
--- Test with pred_type=prob
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample',
- 'id',
- 'x',
- 'cifar10_predict',
- 'prob',
- FALSE);
-
-SELECT assert(UPPER(pg_typeof(prob_0)::TEXT) =
- 'DOUBLE PRECISION', 'column prob_0 should be double precision type')
- FROM cifar10_predict;
-
-SELECT assert(UPPER(pg_typeof(prob_1)::TEXT) =
- 'DOUBLE PRECISION', 'column prob_1 should be double precision type')
- FROM cifar10_predict;
-
-SELECT assert(COUNT(*)=3, 'Predict out table must have exactly three cols.')
-FROM pg_attribute
-WHERE attrelid='cifar10_predict'::regclass AND attnum>0;
-
--- Tests with text class values:
--- Create a new table using the text based column for dep var.
-DROP TABLE IF EXISTS cifar_10_sample_text_batched;
-m4_changequote(`<!', `!>')
-CREATE TABLE cifar_10_sample_text_batched AS
- SELECT buffer_id, independent_var, dependent_var,
- independent_var_shape, dependent_var_shape
- m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!, __dist_key__ !>)
- FROM cifar_10_sample_batched m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!DISTRIBUTED BY (__dist_key__)!>);
-
--- Insert a new row with NULL as the dependent var (one-hot encoded)
-UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0;
-UPDATE cifar_10_sample_text_batched
- SET dependent_var = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1;
-INSERT INTO cifar_10_sample_text_batched(m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) buffer_id, independent_var, dependent_var, independent_var_shape, dependent_var_shape)
- SELECT m4_ifdef(<!__POSTGRESQL__!>, <!!>, <! __dist_key__, !>) 2 AS buffer_id, independent_var,
- convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS dependent_var,
- independent_var_shape, dependent_var_shape
- FROM cifar_10_sample_batched WHERE cifar_10_sample_batched.buffer_id=0;
-UPDATE cifar_10_sample_text_batched SET dependent_var_shape = ARRAY[1,5];
-m4_changequote(<!`!>,<!'!>)
-
--- Create the necessary summary table for the batched input.
-DROP TABLE IF EXISTS cifar_10_sample_text_batched_summary;
-CREATE TABLE cifar_10_sample_text_batched_summary(
- source_table text,
- output_table text,
- dependent_varname text,
- independent_varname text,
- dependent_vartype text,
- class_values text[],
- buffer_size integer,
- normalizing_const numeric);
-INSERT INTO cifar_10_sample_text_batched_summary values (
- 'cifar_10_sample',
- 'cifar_10_sample_text_batched',
- 'y_text',
- 'x',
- 'text',
- ARRAY[NULL,'cat','dog',NULL,NULL],
- 1,
- 255.0);
-
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_text_batched',
- 'keras_saved_out',
- 'model_arch',
- 2,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
-
--- Predict with pred_type=prob
-DROP TABLE IF EXISTS cifar_10_sample_text;
-CREATE TABLE cifar_10_sample_text AS
- SELECT id, x, y_text
- FROM cifar_10_sample;
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample_text',
- 'id',
- 'x',
- 'cifar10_predict',
- 'prob',
- FALSE);
-
--- Validate the output datatype of newly created prediction columns
--- for prediction type = 'prob' and class_values 'TEXT' with NULL as a valid
--- class_values
-SELECT assert(UPPER(pg_typeof(prob_cat)::TEXT) =
- 'DOUBLE PRECISION', 'column prob_cat should be double precision type')
-FROM cifar10_predict;
-
-SELECT assert(UPPER(pg_typeof(prob_dog)::TEXT) =
- 'DOUBLE PRECISION', 'column prob_dog should be double precision type')
-FROM cifar10_predict;
-
-SELECT assert(UPPER(pg_typeof("prob_NULL")::TEXT) =
- 'DOUBLE PRECISION', 'column prob_NULL should be double precision type')
-FROM cifar10_predict;
-
--- Must have exactly 4 cols (3 for class_values and 1 for id)
-SELECT assert(COUNT(*)=4, 'Predict out table must have exactly four cols.')
-FROM pg_attribute
-WHERE attrelid='cifar10_predict'::regclass AND attnum>0;
-
--- Predict with pred_type=response
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample_text',
- 'id',
- 'x',
- 'cifar10_predict',
- 'response',
- FALSE);
-
--- Validate the output datatype of newly created prediction columns
--- for prediction type = 'response' and class_values 'TEXT' with NULL
--- as a valid class_values
-SELECT assert(UPPER(pg_typeof(estimated_y_text)::TEXT) =
- 'TEXT', 'prediction column should be TEXT type')
-FROM cifar10_predict LIMIT 1;
-
--- Tests where the assumption is user has one-hot encoded, so class_values
--- in input summary table will be NULL.
-UPDATE keras_saved_out_summary SET class_values=NULL;
-
--- Predict with pred_type=prob
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample_text',
- 'id',
- 'x',
- 'cifar10_predict',
- 'prob',
- FALSE);
-
--- Validate the output datatype of newly created prediction column
--- for prediction type = 'response' and class_value = NULL
--- Returns: Array of probabilities for user's one-hot encoded data
-SELECT assert(UPPER(pg_typeof(prob)::TEXT) =
- 'DOUBLE PRECISION[]', 'column prob should be double precision[] type')
-FROM cifar10_predict LIMIT 1;
-
--- Predict with pred_type=response
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample_text',
- 'id',
- 'x',
- 'cifar10_predict',
- 'response',
- FALSE);
-
--- Validate the output datatype of newly created prediction column
--- for prediction type = 'response' and class_value = NULL
--- Returns: Index of class value in user's one-hot encoded data with
--- highest probability
-SELECT assert(UPPER(pg_typeof(estimated_y_text)::TEXT) =
- 'TEXT', 'column estimated_y_text should be text type')
-FROM cifar10_predict LIMIT 1;
-
--- Test predict with INTEGER class_values
--- with NULL as a valid class value
--- Update output_summary table to reflect
--- class_values {NULL,0,1,4,5} and dependent_vartype is SMALLINT
-UPDATE keras_saved_out_summary
-SET dependent_varname = 'y',
- class_values = ARRAY[NULL,0,1,4,5]::INTEGER[],
- dependent_vartype = 'smallint';
--- Predict with pred_type=prob
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample',
- 'id',
- 'x',
- 'cifar10_predict',
- 'prob',
- FALSE);
-
--- Validate the output datatype of newly created prediction column
--- for prediction type = 'prob' and class_values 'INT' with NULL
--- as a valid class_values
-SELECT assert(UPPER(pg_typeof("prob_NULL")::TEXT) =
- 'DOUBLE PRECISION', 'column prob_NULL should be double precision type')
-FROM cifar10_predict;
-
--- Must have exactly 6 cols (5 for class_values and 1 for id)
-SELECT assert(COUNT(*)=6, 'Predict out table must have exactly six cols.')
-FROM pg_attribute
-WHERE attrelid='cifar10_predict'::regclass AND attnum>0;
-
--- Predict with pred_type=response
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample',
- 'id',
- 'x',
- 'cifar10_predict',
- 'response',
- FALSE);
-
--- Validate the output datatype of newly created prediction column
--- for prediction type = 'response' and class_values 'TEXT' with NULL
--- as a valid class_values
--- Returns: class_value with highest probability
-SELECT assert(UPPER(pg_typeof(estimated_y)::TEXT) =
- 'SMALLINT', 'prediction column should be smallint type')
-FROM cifar10_predict;
-
--- Predict with correctly shaped data, must go thru.
--- Update output_summary table to reflect
--- class_values, num_classes and model_id for shaped data
-DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
-SELECT madlib_keras_fit(
- 'cifar_10_sample_test_shape_batched',
- 'keras_saved_out',
- 'model_arch',
- 3,
- $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
- $$ batch_size=2, epochs=1, verbose=0 $$::text,
- 3);
-
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample_test_shape',
- 'id',
- 'x',
- 'cifar10_predict',
- 'prob',
- FALSE);
-
--- Prediction with incorrectly shaped data must error out.
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT assert(trap_error($TRAP$SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample',
- 'id',
- 'x',
- 'cifar10_predict',
- 'prob',
- FALSE);$TRAP$) = 1,
- 'Input shape is (32, 32, 3) but model was trained with (3, 32, 32). Should have failed.');
-
--- Test model_arch is retrieved from model data table and not model architecture
-DROP TABLE IF EXISTS model_arch;
-DROP TABLE IF EXISTS cifar10_predict;
-SELECT madlib_keras_predict(
- 'keras_saved_out',
- 'cifar_10_sample_test_shape',
- 'id',
- 'x',
- 'cifar10_predict',
- 'prob',
- FALSE);
-
--- Test multi model
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
-)
-
-m4_changequote(`<!', `!>')
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
-
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)', metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.0001)', metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=50, epochs=1$$
- ]
-);
-
-DROP TABLE IF EXISTS iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT setseed(0);
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 6,
- FALSE
-);
-
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib_keras_predict(
- 'iris_multiple_model', -- model
- 'iris_train', -- test_table
- 'id', -- id column
- 'attributes', -- independent var
- 'iris_predict', -- output table
- 'response', -- prediction type
- NULL, -- use gpus
- 2 -- mst_key to use
- );
-
-SELECT assert(relative_error(test_accuracy, training_metrics_final) < 0.1,
- 'Predict output validation failed.')
-FROM iris_multiple_model_info i,
-(SELECT count(*)/(150*0.8) AS test_accuracy FROM
- (SELECT iris_train.class_text AS actual, iris_predict.estimated_class_text AS estimated
- FROM iris_predict INNER JOIN iris_train
- ON iris_train.id=iris_predict.id)q
- WHERE q.actual=q.estimated) q2
-WHERE i.mst_key = 2;
-!>)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in
deleted file mode 100644
index 12dee6f..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in
+++ /dev/null
@@ -1,137 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
-)
-
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib_keras_fit('iris_data_packed', -- source table
- 'iris_model', -- model output table
- 'iris_model_arch', -- model arch table
- 1, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 3); -- num_iterations
-
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib_keras_predict('iris_model', -- model
- 'iris_test', -- test_table
- 'id', -- id column
- 'attributes', -- independent var
- 'iris_predict' -- output table
- );
-
--- Copy weights that were learnt from the previous run, for transfer
--- learning. Copy it now, because using warm_start will overwrite it.
-UPDATE iris_model_arch set model_weights = (select model_weights from iris_model) WHERE model_id = 2;
-
--- class_values not NULL, pred_type is response
-DROP TABLE IF EXISTS iris_predict_byom;
-SELECT madlib_keras_predict_byom(
- 'iris_model_arch',
- 2,
- 'iris_test',
- 'id',
- 'attributes',
- 'iris_predict_byom',
- 'response',
- NULL,
- ARRAY['Iris-setosa', 'Iris-versicolor',
- 'Iris-virginica']
- );
-
-SELECT assert(
- p0.estimated_class_text = p1.estimated_dependent_var,
- 'Predict byom failure for non null class value and response pred_type.')
-FROM iris_predict AS p0, iris_predict_byom AS p1
-WHERE p0.id=p1.id;
-SELECT assert(UPPER(pg_typeof(estimated_dependent_var)::TEXT) = 'TEXT',
- 'Predict byom failure for non null class value and response pred_type.
- Expeceted estimated_dependent_var to be of type TEXT')
-FROM iris_predict_byom LIMIT 1;
-
--- class_values NULL, pred_type is NULL (response)
-DROP TABLE IF EXISTS iris_predict_byom;
-SELECT madlib_keras_predict_byom(
- 'iris_model_arch',
- 2,
- 'iris_test',
- 'id',
- 'attributes',
- 'iris_predict_byom'
- );
-SELECT assert(
- p1.estimated_dependent_var IN ('0', '1', '2'),
- 'Predict byom failure for null class value and null pred_type.')
-FROM iris_predict_byom AS p1;
-SELECT assert(UPPER(pg_typeof(estimated_dependent_var)::TEXT) = 'TEXT',
- 'Predict byom failure for non null class value and response pred_type.
- Expeceted estimated_dependent_var to be of type TEXT')
-FROM iris_predict_byom LIMIT 1;
-
--- class_values not NULL, pred_type is prob
-DROP TABLE IF EXISTS iris_predict_byom;
-SELECT madlib_keras_predict_byom(
- 'iris_model_arch',
- 2,
- 'iris_test',
- 'id',
- 'attributes',
- 'iris_predict_byom',
- 'prob',
- NULL,
- ARRAY['Iris-setosa', 'Iris-versicolor',
- 'Iris-virginica'],
- 1.0
- );
-
-SELECT assert(
- (p1."prob_Iris-setosa" + p1."prob_Iris-virginica" + p1."prob_Iris-versicolor") - 1 < 1e-6,
- 'Predict byom failure for non null class value and prob pred_type.')
-FROM iris_predict_byom AS p1;
-SELECT assert(UPPER(pg_typeof("prob_Iris-setosa")::TEXT) = 'DOUBLE PRECISION',
- 'Predict byom failure for non null class value and prob pred_type.
- Expeceted "prob_Iris-setosa" to be of type DOUBLE PRECISION')
-FROM iris_predict_byom LIMIT 1;
-
--- class_values NULL, pred_type is prob
-DROP TABLE IF EXISTS iris_predict_byom;
-SELECT madlib_keras_predict_byom(
- 'iris_model_arch',
- 2,
- 'iris_test',
- 'id',
- 'attributes',
- 'iris_predict_byom',
- 'prob',
- NULL,
- NULL
- );
-SELECT assert(
- (prob[1] + prob[2] + prob[3]) - 1 < 1e-6,
- 'Predict byom failure for null class value and prob pred_type.')
-FROM iris_predict_byom;
-SELECT assert(UPPER(pg_typeof(prob)::TEXT) = 'DOUBLE PRECISION[]',
- 'Predict byom failure for null class value and prob pred_type. Expeceted prob to
- be of type DOUBLE PRECISION[]')
-FROM iris_predict_byom LIMIT 1;
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in
deleted file mode 100644
index 43ad38b..0000000
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in
+++ /dev/null
@@ -1,355 +0,0 @@
-/* ---------------------------------------------------------------------*//**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- *//* ---------------------------------------------------------------------*/
-
-m4_include(`SQLCommon.m4')
-
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
-)
--------------- Warm start test (along with schema qualified output table) -------------------------
-CREATE SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__;
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib_keras_fit('iris_data_packed', -- source table
- '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model', -- model output table
- 'iris_model_arch', -- model arch table
- 1, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 5, -- num_iterations
- NULL, NULL,
- 1 -- metrics_compute_frequency
- );
-
--- Test that our code is indeed learning something and not broken. The loss
--- from the first iteration should be less than the 5th, while the accuracy
--- must be greater.
-SELECT assert(
- array_upper(training_loss, 1) = 5 AND
- array_upper(training_metrics, 1) = 5,
- 'metrics compute frequency must be 1.')
-FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary;
-
-SELECT assert(
- training_loss[5]-training_loss[1] < 0.1 AND
- training_metrics[5]-training_metrics[1] > -0.1,
- 'The loss and accuracy should have improved with more iterations.'
-)
-FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary;
-
--- Make a copy of the loss and metrics array, to compare it with runs after
--- warm start and transfer learning.
-DROP TABLE IF EXISTS iris_model_first_run;
-CREATE TABLE iris_model_first_run AS
-SELECT training_loss_final, training_metrics_final
-FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary;
-
--- Copy weights that were learnt from the previous run, for transfer
--- learning. Copy it now, because using warm_start will overwrite it.
-UPDATE iris_model_arch set model_weights = (select model_weights from __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model)
-WHERE model_id = 2;
-
--- Warm start test
-SELECT madlib_keras_fit('iris_data_packed', -- source table
- '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model', -- model output table
- 'iris_model_arch', -- model arch table
- 2, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 2, -- num_iterations,
- NULL, NULL, 1,
- TRUE -- warm start
- );
-
-SELECT assert(
- array_upper(training_loss, 1) = 2 AND
- array_upper(training_metrics, 1) = 2,
- 'metrics compute frequency must be 1.')
-FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary;
-
-SELECT assert(
- abs(first.training_loss_final-second.training_loss[1]) < 1e-6 AND
- abs(first.training_loss_final-second.training_loss[2]) < 1e-6 AND
- abs(first.training_metrics_final-second.training_metrics[1]) < 1e-10 AND
- abs(first.training_metrics_final-second.training_metrics[2]) < 1e-10,
- 'warm start test failed because training loss and metrics don''t match the expected value from the previous run of keras fit.')
-FROM iris_model_first_run AS first, __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary AS second;
-
-DROP SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__ CASCADE;
-
----------------- end Warm start test ----------------------------------------------------
-
----------------- Transfer learning test ----------------------------------------------------
-DROP TABLE IF EXISTS iris_model_transfer, iris_model_transfer_summary;
-SELECT madlib_keras_fit('iris_data_packed', -- source table
- 'iris_model_transfer', -- model output table
- 'iris_model_arch', -- model arch table
- 2, -- model arch id
- $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params
- $$ batch_size=5, epochs=3 $$, -- fit_params
- 2,
- NULL, NULL, 1
- );
-
-SELECT assert(
- array_upper(training_loss, 1) = 2 AND
- array_upper(training_metrics, 1) = 2,
- 'metrics compute frequency must be 1.')
-FROM iris_model_transfer_summary;
-
-SELECT assert(
- abs(first.training_loss_final-second.training_loss[1]) < 1e-6 AND
- abs(first.training_loss_final-second.training_loss[2]) < 1e-6 AND
- abs(first.training_metrics_final-second.training_metrics[1]) < 1e-10 AND
- abs(first.training_metrics_final-second.training_metrics[2]) < 1e-10,
- 'Transfer learning test failed because training loss and metrics don''t match the expected value.')
-FROM iris_model_first_run AS first,
-iris_model_transfer_summary AS second;
-
--- Rerun the iris setup to discard the changes
-\i m4_regexp(MODULE_PATHNAME,
- `\(.*\)libmadlib\.so',
- `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in'
-)
-
-m4_changequote(`<!', `!>')
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
-
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1,2],
- ARRAY[
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$
- ]
-);
-
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT setseed(0);
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 3,
- FALSE, NULL, 1
-);
-
-DROP TABLE IF EXISTS iris_model_first_run;
-CREATE TABLE iris_model_first_run AS
-SELECT mst_key, model_id, training_loss, training_metrics,
- training_loss_final, training_metrics_final
-FROM iris_multiple_model_info;
-
--- warm start for fit multiple model
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 3,
- FALSE,
- NULL, 1,
- TRUE -- warm_start
-);
-
--- Test that when warm_start is TRUE, all the output tables are persistent(not unlogged)
-SELECT assert(MADLIB_SCHEMA.is_table_unlogged('iris_multiple_model') = false, 'Model output table is unlogged');
-SELECT assert(MADLIB_SCHEMA.is_table_unlogged('iris_multiple_model_summary') = false, 'Model summary output table is unlogged');
-SELECT assert(MADLIB_SCHEMA.is_table_unlogged('iris_multiple_model_info') = false, 'Model info output table is unlogged');
-
-
-SELECT assert(
- array_upper(training_loss, 1) = 3 AND
- array_upper(training_metrics, 1) = 3,
- 'metrics compute frequency must be 1.')
-FROM iris_multiple_model_info;
-
-SELECT assert(
- abs(first.training_loss_final-second.training_loss[1]) < 1e-6 AND
- abs(first.training_loss_final-second.training_loss[2]) < 1e-6 AND
- abs(first.training_metrics_final-second.training_metrics[1]) < 1e-10 AND
- abs(first.training_metrics_final-second.training_metrics[2]) < 1e-10,
- 'warm start test failed because training loss and metrics don''t match the expected value from the previous run of keras fit.')
-FROM iris_model_first_run AS first, iris_multiple_model_info AS second
-WHERE first.mst_key = second.mst_key AND first.model_id = 2;
-
--- warm start with different mst tables
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1],
- ARRAY[
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$,
- $$batch_size=10,epochs=1$$,
- $$batch_size=15,epochs=1$$,
- $$batch_size=20,epochs=1$$
- ]
-);
-
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT setseed(0);
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 3,
- FALSE, NULL, 1
-);
-
-DROP TABLE IF EXISTS iris_model_first_run;
-CREATE TABLE iris_model_first_run AS
-SELECT mst_key, model_id, training_loss, training_metrics,
- training_loss_final, training_metrics_final
-FROM iris_multiple_model_info;
-
-DELETE FROM mst_table WHERE mst_key = 4;
-
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 3,
- FALSE, NULL, 1,
- TRUE);
-
-SELECT assert(
- 4 NOT IN (SELECT mst_key FROM iris_multiple_model),
- 'mst_key 4 should not be in the model table since it has been removed from mst_table');
-
-SELECT assert(
- 4 NOT IN (SELECT mst_key FROM iris_multiple_model_info),
- 'mst_key 4 should not be in the info table since it has been removed from mst_table');
-
-INSERT INTO mst_table SELECT 5 AS mst_key, model_id, compile_params,
- 'batch_size=18, epochs=1' FROM mst_table WHERE mst_key = 1;
-
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 3,
- FALSE,
- NULL, 1,
- TRUE -- warm_start
-);
--- The default value of the guc 'dev_opt_unsafe_truncate_in_subtransaction' is 'off'
--- but we change it to 'on' in fit_multiple.py. Assert that the value is
--- reset after calling fit_multiple
-SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('dev_opt_unsafe_truncate_in_subtransaction', 'off') END;
-
-SELECT assert(
- 5 IN (SELECT mst_key FROM iris_multiple_model),
- 'mst_key 5 should be in the model table since it has been added to mst_table');
-
-SELECT assert(
- 5 IN (SELECT mst_key FROM iris_multiple_model_info),
- 'mst_key 5 should be in the info table since it has been added to mst_table');
-
--- Transfer learning tests
-
--- Load the same arch again so that we can compare transfer learning results
-SELECT load_keras_model('iris_model_arch', -- Output table,
-$$
-{
-"class_name": "Sequential",
-"keras_version": "2.1.6",
-"config":
- [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling",
- "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}},
- "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null,
- "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true,
- "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
- "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true,
- "activity_regularizer": null}}, {"class_name": "Dense",
- "config": {"kernel_initializer": {"class_name": "VarianceScaling",
- "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}},
- "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null,
- "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null,
- "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true,
- "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer":
- {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0,
- "seed": 0, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null,
- "bias_regularizer": null, "bias_constraint": null, "activation": "softmax",
- "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
- "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}],
- "backend": "tensorflow"}
-$$
-);
-
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT load_model_selection_table(
- 'iris_model_arch',
- 'mst_table',
- ARRAY[1,3],
- ARRAY[
- $$loss='categorical_crossentropy',optimizer='Adam(lr=0.00001)',metrics=['accuracy']$$,
- $$loss='categorical_crossentropy', optimizer='Adam(lr=0.00002)',metrics=['accuracy']$$
- ],
- ARRAY[
- $$batch_size=5,epochs=1$$
- ]
-);
-
--- TODO we need to drop iris_multiple_model as well as iris_multiple_model
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT setseed(0);
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 3,
- FALSE, NULL, 1
-);
-
-UPDATE iris_model_arch
-SET model_weights = (SELECT model_weights FROM iris_multiple_model WHERE mst_key=1)
-WHERE model_id = 1;
-
-DROP TABLE IF EXISTS iris_model_first_run;
-CREATE TABLE iris_model_first_run AS
-SELECT mst_key, model_id, training_loss, training_metrics,
- training_loss_final, training_metrics_final
-FROM iris_multiple_model_info;
-
-DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info;
-SELECT madlib_keras_fit_multiple_model(
- 'iris_data_packed',
- 'iris_multiple_model',
- 'mst_table',
- 3,
- FALSE, NULL, 1
-);
-
-SELECT assert(
- (first.training_loss_final-second.training_loss_final) > 1e-6,
- 'Transfer learning test failed because training loss and metrics don''t match the expected value.')
-FROM iris_model_first_run AS first, iris_multiple_model_info AS second
-WHERE first.mst_key = second.mst_key AND first.model_id = 1;
-
-!>)
-
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/plpy_mock.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/plpy_mock.py_in
deleted file mode 100644
index dd18649..0000000
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/plpy_mock.py_in
+++ /dev/null
@@ -1,43 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-m4_changequote(`<!', `!>')
-def __init__(self):
- pass
-
-def error(message):
- raise PLPYException(message)
-
-def execute(query):
- pass
-
-def warning(query):
- pass
-
-def info(query):
- print query
-
-
-class PLPYException(Exception):
- def __init__(self, message):
- super(PLPYException, self).__init__()
- self.message = message
-
- def __str__(self):
- return repr(self.message)
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in
deleted file mode 100644
index d2e14cd..0000000
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in
+++ /dev/null
@@ -1,383 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import sys
-import numpy as np
-from os import path
-
-# Add modules to the pythonpath.
-sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))))
-sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))
-
-import unittest
-from mock import *
-import plpy_mock as plpy
-
-m4_changequote(`<!', `!>')
-
-class InputPreProcessorDLTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy,
- 'utilities.mean_std_dev_calculator': Mock(),
- }
- # we need to use MagicMock() instead of Mock() for the plpy.execute mock
- # to be able to iterate on the return value
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
-
- self.default_schema_madlib = "madlib"
- self.default_source_table = "source"
- self.default_output_table = "output"
- self.default_dep_var = "depvar"
- self.default_ind_var = "indvar"
- self.default_buffer_size = 5
- self.default_normalizing_const = 1.0
- self.default_num_classes = None
- self.default_distribution_rules = "all_segments"
- self.default_module_name = "dummy"
-
- import deep_learning.input_data_preprocessor
- self.module = deep_learning.input_data_preprocessor
- import utilities.minibatch_preprocessing
- self.util_module = utilities.minibatch_preprocessing
- import utilities.control
- self.control_module = utilities.control
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- self.module.validate_module_input_params = Mock()
- self.module.get_distinct_col_levels = Mock(return_value = [0,22,100])
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_input_preprocessor_dl_executes_query(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- self.control_module.OptimizerControl.__enter__ = Mock()
- self.control_module.OptimizerControl.optimizer_control = True
- self.control_module.OptimizerControl.optimizer_enabled = True
- preprocessor_obj = self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- "input",
- "out",
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
- preprocessor_obj.input_preprocessor_dl()
-
- def test_input_preprocessor_null_buffer_size_executes_query(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- self.control_module.OptimizerControl.__enter__ = Mock()
- self.control_module.OptimizerControl.optimizer_control = True
- self.control_module.OptimizerControl.optimizer_enabled = True
- preprocessor_obj = self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- "input",
- "out",
- self.default_dep_var,
- self.default_ind_var,
- None,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
- self.util_module.MiniBatchBufferSizeCalculator.calculate_default_buffer_size = Mock(return_value = 5)
- preprocessor_obj.input_preprocessor_dl()
-
- def test_input_preprocessor_multiple_dep_var_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- "y1,y2",
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_multiple_indep_var_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- "x1,x2",
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_buffer_size_zero_fails(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- 0,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_negative_buffer_size_fails(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- -1,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_invalid_indep_vartype_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_invalid_dep_vartype_raises_exception(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_normalizing_const_zero_fails(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- 0,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_input_preprocessor_negative_normalizing_const_fails(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- with self.assertRaises(plpy.PLPYException):
- self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- -1,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
-
- def test_get_one_hot_encoded_dep_var_expr_null_val(self):
- self.module.get_expr_type = Mock(side_effect = ['smallint[]', 'text'])
- self.module.get_distinct_col_levels = Mock(return_value = ["NULL", "'a'"])
- obj = self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
- obj.dependent_levels = ["NULL", "'a'"]
- dep_var_array_expr = obj.get_one_hot_encoded_dep_var_expr()
- self.assertEqual("array[({0}) is not distinct from null, " \
- "({0}) is not distinct from 'a']::integer[]::smallint[]".
- format(self.default_dep_var),
- dep_var_array_expr.lower())
-
- def test_get_one_hot_encoded_dep_var_expr_numeric_array_val(self):
- self.module.get_expr_type = Mock(side_effect = ['smallint[]', 'integer[]'])
- obj = self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
- dep_var_array_expr = obj.get_one_hot_encoded_dep_var_expr()
- self.assertEqual("{0}::smallint[]".
- format(self.default_dep_var),
- dep_var_array_expr.lower())
-
- def test_validate_num_classes_none(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text'])
- obj = self.module.InputDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- None,
- self.default_distribution_rules,
- self.default_module_name)
- obj.dependent_levels = ["dummy"]
- self.assertEqual(0, obj.padding_size)
-
- def test_validate_num_classes_greater(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text'])
- self.module._get_dependent_levels = Mock(return_value = ["'a'", "'b'", "'c'"])
- obj = self.module.TrainingDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- 5,
- self.default_distribution_rules)
- obj._set_one_hot_encoding_variables()
- self.assertEqual(2, obj.padding_size)
-
- def test_validate_num_classes_lesser(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text'])
- self.module.dependent_levels = Mock(return_value = ["'a'", "'b'", "'c'"])
- obj = self.module.TrainingDataPreprocessorDL(
- self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- 2,
- self.default_distribution_rules)
- with self.assertRaises(plpy.PLPYException):
- obj._set_one_hot_encoding_variables()
-
- def test_validate_distribution_table(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
-
- obj = self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
- self.module.input_tbl_valid = Mock()
- self.module.is_var_valid = Mock()
- self.plpy_mock_execute.side_effect = [
- [{'dbids': [2,3,4]}],
- [{'dbids': [3,4], 'c1': 2, 'c2': 2}]
- ]
-
- obj._validate_distribution_table()
-
- def test_validate_distribution_table_dup(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
-
- obj = self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
- self.module.input_tbl_valid = Mock()
- self.module.is_var_valid = Mock()
- self.plpy_mock_execute.side_effect = [
- [{'dbids': [2,3,4]}],
- [{'dbids': [3,3], 'c1': 2, 'c2': 1}]
- ]
- with self.assertRaises(plpy.PLPYException) as error:
- obj._validate_distribution_table()
- self.assertIn('duplicate', str(error.exception).lower())
-
- def test_validate_distribution_table_invalid(self):
- self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]'])
- obj = self.module.InputDataPreprocessorDL(self.default_schema_madlib,
- self.default_source_table,
- self.default_output_table,
- self.default_dep_var,
- self.default_ind_var,
- self.default_buffer_size,
- self.default_normalizing_const,
- self.default_num_classes,
- self.default_distribution_rules,
- self.default_module_name)
- self.module.input_tbl_valid = Mock()
- self.module.is_var_valid = Mock()
- self.plpy_mock_execute.side_effect = [
- [{'dbids': [2,3,4]}],
- [{'dbids': [3,30], 'c1': 2, 'c2': 2}]
- ]
- with self.assertRaises(plpy.PLPYException) as error:
- obj._validate_distribution_table()
- self.assertIn('invalid', str(error.exception).lower())
-
-
-if __name__ == '__main__':
- unittest.main()
-
-# ---------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
deleted file mode 100644
index ed3e0da..0000000
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
+++ /dev/null
@@ -1,1681 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-m4_changequote(`<!', `!>')
-
-import sys
-import numpy as np
-import os
-from os import path
-# Add convex module to the pythonpath.
-sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))))
-sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))
-
-import keras
-from keras.models import *
-from keras.layers import *
-import unittest
-from mock import *
-import plpy_mock as plpy
-
-# helper for multiplying array by int
-def mult(k,arr):
- return [ k*a for a in arr ]
-
-class MadlibKerasFitTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras
- self.subject = madlib_keras
-
- self.model = Sequential()
- self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu',
- input_shape=(1,1,1,), padding='same'))
- self.model.add(Flatten())
-
- self.compile_params = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']"
- self.fit_params = "batch_size=1, epochs=1"
- self.model_weights = [3,4,5,6]
- self.serialized_weights = np.array(self.model_weights, dtype=np.float32
- ).tostring()
- self.dist_key_mapping = [0,1,2]
- self.accessible_gpus_for_seg = [0]
-
- self.independent_var_real = [[[[0.5]]]] * 10
- self.dependent_var_int = [[0,1]] * 10
-
- # Params as bytea
- self.independent_var = np.array(self.independent_var_real, dtype=np.float32).tobytes()
- self.dependent_var = np.array(self.dependent_var_int, dtype=np.int16).tobytes()
-
- self.independent_var_shape = [10,1,1,1]
- self.dependent_var_shape = [10,2]
- # We test on segment 0, which has 3 buffers filled with 10 identical
- # images each, or 30 images total
- self.total_images_per_seg = [3*len(self.dependent_var_int),20,40]
-
- self.dummy_prev_weights = 'dummy weights'
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def _test_fit_transition_first_buffer_pass(self, is_platform_pg):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
- self.subject.is_platform_pg = Mock(return_value = is_platform_pg)
- starting_image_count = 0
- ending_image_count = len(self.dependent_var_int)
-
- # last iteration Call
- previous_state = np.array(self.model_weights, dtype=np.float32)
-
- k = {'SD': {}}
-
- new_state = self.subject.fit_transition(
- None, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), self.compile_params, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, previous_state.tostring(), True, **k)
-
- image_count = new_state
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must not get called for the first buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
- self.assertTrue(k['SD']['segment_model'])
-
- # Non-last iteration Call
- self.subject.K.set_session.reset_mock()
- self.subject.K.clear_session.reset_mock()
- previous_state = np.array(self.model_weights, dtype=np.float32)
-
- k = {'SD' : {}}
-
- new_state = self.subject.fit_transition(
- None, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), self.compile_params, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, previous_state.tostring(), False, **k)
-
- image_count = new_state
-
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must not get called for the first buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
- self.assertTrue(k['SD']['segment_model'])
-
- def test_fit_transition_multiple_model_first_buffer_pass(self):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
- starting_image_count = 0
- ending_image_count = len(self.dependent_var_int)
-
- previous_weights = np.array(self.model_weights, dtype=np.float32)
-
- k = {'SD': {}}
-
- new_state = self.subject.fit_transition(
- None, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), self.compile_params, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, previous_weights.tostring(), True, True, **k)
-
- image_count = new_state
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session must not be called for the first buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
- self.assertTrue(k['SD']['segment_model'])
-
- def _test_fit_transition_middle_buffer_pass(self, is_platform_pg):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
- self.subject.is_platform_pg = Mock(return_value = is_platform_pg)
-
- starting_image_count = len(self.dependent_var_int)
- ending_image_count = starting_image_count + len(self.dependent_var_int)
-
- # last iteration Call
-
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
- k = {'SD': {'segment_model': self.model, 'sess': Mock()}}
-
- state = starting_image_count
- new_state = self.subject.fit_transition(
- state, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), None, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, self.dummy_prev_weights, True, **k)
-
- image_count = new_state
- self.assertEqual(ending_image_count, image_count)
-
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must not get called for the middle buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
-
- # Non-last iteration Call
- self.subject.K.set_session.reset_mock()
- self.subject.K.clear_session.reset_mock()
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
- k = {'SD': {'segment_model': self.model, 'sess': Mock()}}
-
- state = starting_image_count
- new_state = self.subject.fit_transition(
- state, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), None, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, self.dummy_prev_weights, False, **k)
-
- image_count = new_state
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must not get called for the middle buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
-
- def test_fit_transition_multiple_model_middle_buffer_pass(self):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
-
- starting_image_count = len(self.dependent_var_int)
- ending_image_count = starting_image_count + len(self.dependent_var_int)
-
- # last iteration Call
-
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
- k = {'SD': {'segment_model': self.model, 'sess': Mock()}}
-
- state = starting_image_count
- new_state = self.subject.fit_transition(
- state, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), None, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, self.dummy_prev_weights, True, True, **k)
-
- image_count = new_state
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must not get called for the middle buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
-
- def _test_fit_transition_last_buffer_pass(self, is_platform_pg):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
- self.subject.is_platform_pg = Mock(return_value = is_platform_pg)
-
- starting_image_count = 2*len(self.dependent_var_int)
- ending_image_count = starting_image_count + len(self.dependent_var_int)
-
- # last iteration Call
-
- multiplied_weights = mult(self.total_images_per_seg[0],self.model_weights)
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
- k = {'SD': {'segment_model' :self.model, 'sess': Mock()}}
-
- state = starting_image_count
- new_state = self.subject.fit_transition(
- state, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), None, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, self.dummy_prev_weights, True, **k)
-
- state = np.fromstring(new_state, dtype=np.float32)
- image_count = state[0]
- weights = np.rint(state[1:]).astype(np.int)
- self.assertEqual(ending_image_count, image_count)
- # weights should be multiplied by final image count
- self.assertTrue((multiplied_weights == weights).all())
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must get called for the last buffer in gpdb,
- # but not in postgres
- self.assertEqual(1, self.subject.K.clear_session.call_count)
-
- # Non-last iteration Call
- self.subject.K.set_session.reset_mock()
- self.subject.K.clear_session.reset_mock()
-
- multiplied_weights = mult(self.total_images_per_seg[0],self.model_weights)
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
- k = {'SD': {'segment_model' :self.model, 'sess': Mock()}}
-
- state = starting_image_count
- new_state = self.subject.fit_transition(
- state, self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), None, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, self.dummy_prev_weights, False, **k)
-
- state = np.fromstring(new_state, dtype=np.float32)
- image_count = state[0]
- weights = np.rint(state[1:]).astype(np.int)
- self.assertEqual(ending_image_count, image_count)
- # weights should be multiplied by final image count
- self.assertTrue((multiplied_weights == weights).all())
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must get called for the last buffer in gpdb,
- # but not in postgres
- self.assertEqual(0, self.subject.K.clear_session.call_count)
-
- def test_fit_transition_multiple_model_last_buffer_pass(self):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
-
- starting_image_count = 2*len(self.dependent_var_int)
- ending_image_count = starting_image_count + len(self.dependent_var_int)
-
- # last iteration Call
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
- k = {'SD': {'segment_model' :self.model, 'sess': Mock()}}
-
- state = starting_image_count
- new_state = self.subject.fit_transition(
- state , self.dependent_var, self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(), None, self.fit_params, 0,
- self.dist_key_mapping, 0, 4, self.total_images_per_seg, False,
- self.accessible_gpus_for_seg, self.dummy_prev_weights, True, True, **k)
-
- state = np.fromstring(new_state, dtype=np.float32)
- weights = np.rint(state[0:]).astype(np.int)
-
- ## image count should not be added to the final state of
- # fit multiple
- self.assertEqual(len(self.model_weights), len(weights))
-
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # Clear session and sess.close must get called for the last buffer in gpdb,
- # but not in postgres
- self.assertEqual(1, self.subject.K.clear_session.call_count)
-
- def test_fit_transition_first_buffer_pass_pg(self):
- self._test_fit_transition_first_buffer_pass(True)
-
- def test_fit_transition_first_buffer_pass_gpdb(self):
- self._test_fit_transition_first_buffer_pass(False)
-
- def test_fit_transition_middle_buffer_pass_pg(self):
- self._test_fit_transition_middle_buffer_pass(True)
-
- def test_fit_transition_middle_buffer_pass_gpdb(self):
- self._test_fit_transition_middle_buffer_pass(False)
-
- def test_fit_transition_last_buffer_pass_pg(self):
- self._test_fit_transition_last_buffer_pass(True)
-
- def test_fit_transition_last_buffer_pass_gpdb(self):
- self._test_fit_transition_last_buffer_pass(False)
-
- def test_fit_transition_first_tuple_none_ind_var_dep_var(self):
- k = {}
- self.assertEqual('dummy_state',
- self.subject.fit_transition('dummy_state', [0], None,
- 'noshape', 'noshape',
- 'dummy_model_json', "foo", "bar",
- 1, [0,1,2], 0, 4, [3,3,3], False,
- [0], 'dummy_prev_state', **k))
- self.assertEqual('dummy_state',
- self.subject.fit_transition('dummy_state', None, [[0.5]],
- 'noshape', 'noshape',
- 'dummy_model_json', "foo", "bar",
- 1, [0,1,2], 0, 4, [3,3,3], False,
- [0], 'dummy_prev_state', **k))
- self.assertEqual('dummy_state',
- self.subject.fit_transition('dummy_state', None, None,
- 'noshape', 'noshape',
- 'dummy_model_json', "foo", "bar",
- 1, [0,1,2], 0, 4, [3,3,3], False,
- [0], 'dummy_prev_state', **k))
-
- def test_fit_merge(self):
- image_count = self.total_images_per_seg[0]
- state1 = [image_count]
- state1.extend(mult(3,self.model_weights))
- state1 = np.array(state1, dtype=np.float32)
- state2 = [image_count+30]
- state2.extend(mult(2,self.model_weights))
- state2 = np.array(state2, dtype=np.float32)
- merged_state = self.subject.fit_merge(state1.tostring(),state2.tostring())
- state = np.fromstring(merged_state, dtype=np.float32)
- image_count_total = state[0]
- weights = np.rint(state[1:]).astype(np.int)
-
- self.assertEqual( 2*image_count+30 , image_count_total )
- self.assertTrue( (mult(5,self.model_weights) == weights).all())
-
- def test_fit_merge_none_first(self):
- image_count = self.total_images_per_seg[0]
- input_state = [image_count]
- input_state.extend(self.model_weights)
- input_state = np.array(input_state, dtype=np.float32)
- merged_state = self.subject.fit_merge(None, input_state.tostring())
- state = np.fromstring(merged_state, dtype=np.float32)
- image_count_total = state[0]
- weights = np.rint(state[1:]).astype(np.int)
-
- self.assertEqual(image_count, image_count_total)
- self.assertTrue((self.model_weights == weights).all())
-
- def test_fit_merge_none_second(self):
- image_count = self.total_images_per_seg[0]
- input_state = [image_count]
- input_state.extend(self.model_weights)
- input_state = np.array(input_state, dtype=np.float32)
- merged_state = self.subject.fit_merge(input_state.tostring(), None)
- state = np.fromstring(merged_state, dtype=np.float32)
- image_count_total = state[0]
- weights = np.rint(state[1:]).astype(np.int)
-
- self.assertEqual(image_count, image_count_total)
- self.assertTrue((self.model_weights == weights).all())
-
- def test_fit_merge_both_none(self):
- result = self.subject.fit_merge(None,None)
- self.assertEqual(None, result)
-
- def test_fit_final(self):
- image_count = self.total_images_per_seg[0]
- input_state = [image_count]
- input_state.extend(mult(image_count,self.model_weights))
- input_state = np.array(input_state, dtype=np.float32)
-
- output_state = self.subject.fit_final(input_state.tostring())
- output_state = np.fromstring(output_state, dtype=np.float32)
- weights = np.rint(output_state).astype(np.int)
-
- self.assertTrue((self.model_weights == weights).all())
-
- def test_fit_final_none(self):
- result = self.subject.fit_final(None)
- self.assertEqual(result, None)
-
- def test_fit_final_image_count_zero(self):
- input_state = [0]
- input_state.extend(self.model_weights)
- input_state = np.array(input_state, dtype=np.float32)
-
- with self.assertRaises(plpy.PLPYException):
- result = self.subject.fit_final(input_state.tostring())
-
- def test_should_compute_metrics_this_iter_last_iter_case1(self):
- res = self.subject.should_compute_metrics_this_iter(5, 5, 5)
- self.assertEqual(True, res)
- def test_should_compute_metrics_this_iter_last_iter_case2(self):
- res = self.subject.should_compute_metrics_this_iter(5, 3, 5)
- self.assertEqual(True, res)
- def test_should_compute_metrics_this_iter_before_interval(self):
- res = self.subject.should_compute_metrics_this_iter(1, 3, 5)
- self.assertEqual(False, res)
- def test_should_compute_metrics_this_iter_after_interval(self):
- res = self.subject.should_compute_metrics_this_iter(4, 3, 5)
- self.assertEqual(False, res)
- def test_should_compute_metrics_this_iter_at_interval(self):
- res = self.subject.should_compute_metrics_this_iter(3, 3, 5)
- self.assertEqual(True, res)
- def test_should_compute_metrics_this_iter_every_iter(self):
- res = self.subject.should_compute_metrics_this_iter(2, 1, 5)
- self.assertEqual(True, res)
-
-
-class InternalKerasPredictTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras_predict
- self.subject = madlib_keras_predict
-
- self.model = Sequential()
- self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu',
- input_shape=(1,1,1,), padding='same'))
- self.model.add(Flatten())
-
- self.all_seg_ids = [0,1,2]
-
- self.independent_var = [[[240]]]
- self.total_images_per_seg = [3,3,4]
- self.subject.K.set_session = Mock()
- self.subject.clear_keras_session = Mock()
-
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_predict_first_image_pass_gpdb(self):
- self.subject.is_platform_pg = Mock(return_value = False)
- model_weights = [1, 2, 3, 4]
- serialized_weights = np.array(model_weights, dtype=np.float32).tostring()
-
- k = {'SD': {}}
- is_response = True
- result = self.subject.internal_keras_predict(
- self.independent_var, self.model.to_json(),
- serialized_weights, is_response, 255, 0, self.all_seg_ids,
- self.total_images_per_seg, False, 0, 4, **k)
- self.assertEqual(1, len(result))
- self.assertEqual(1, k['SD']['row_count'])
- self.assertEqual(True, 'segment_model_predict' in k['SD'])
-
- def test_predict_middle_image_pass_gpdb(self):
- self.subject.is_platform_pg = Mock(return_value = False)
-
- k = {'SD': { 'row_count': 1}}
- k['SD']['segment_model_predict'] = self.model
- is_response = True
- result = self.subject.internal_keras_predict(
- self.independent_var, None, None, is_response, 255, 0,
- self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k)
- self.assertEqual(1, len(result))
- self.assertEqual(2, k['SD']['row_count'])
- self.assertEqual(True, 'segment_model_predict' in k['SD'])
-
- def test_predict_last_image_pass_gpdb(self):
- self.subject.is_platform_pg = Mock(return_value = False)
- self.model.add(Dense(3))
-
- k = {'SD': { 'row_count': 2}}
- k['SD']['segment_model_predict'] = self.model
- is_response = True
- result = self.subject.internal_keras_predict(
- self.independent_var, None, None, is_response, 255, 0,
- self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k)
- self.assertEqual(1, len(result))
- self.assertEqual(False, 'row_count' in k['SD'])
- self.assertEqual(False, 'segment_model_predict' in k['SD'])
-
- k = {'SD': { 'row_count': 2}}
- k['SD']['segment_model_predict'] = self.model
- is_response = False
- result = self.subject.internal_keras_predict(
- self.independent_var, None, None, is_response, 255, 0,
- self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k)
-
- # we except len(result) to be 3 because we have 3 dense layers in the
- # architecture
- self.assertEqual(3, len(result))
- self.assertEqual(False, 'row_count' in k['SD'])
- self.assertEqual(False, 'segment_model_predict' in k['SD'])
-
-
- def test_predict_error_should_clear_sd(self):
- self.subject.is_platform_pg = Mock(return_value = False)
- self.model.add(Dense(3))
-
- # inject error by passing 0 as the normalizing const so that we get a
- # divide by zero error
- normalizing_const = 0
- k = {'SD':{}}
- is_response = True
- with self.assertRaises(plpy.PLPYException):
- self.subject.internal_keras_predict(
- self.independent_var, None, None, is_response, normalizing_const,
- 0, self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k)
- self.assertEqual(False, 'row_count' in k['SD'])
- self.assertEqual(False, 'segment_model_predict' in k['SD'])
-
-
-class MadlibKerasPredictBYOMTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- self.num_classes = 5
- self.model = Sequential()
- self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu',
- input_shape=(1,1,1,), padding='same'))
- self.model.add(Dense(self.num_classes))
-
- self.pred_type = 'prob'
- self.use_gpus = False
- self.class_values = ['foo', 'bar', 'baaz', 'foo2', 'bar2']
- self.normalizing_const = 255.0
-
- import madlib_keras_predict
- self.module = madlib_keras_predict
- self.module.get_model_arch_weights = Mock(return_value=(
- self.model.to_json(), 'weights'))
- self.module.InputValidator.validate_predict_byom_tables = Mock()
- self.module.InputValidator.validate_input_shape = Mock()
- self.module.BasePredict.call_internal_keras = Mock()
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_predictbyom_defaults_1(self):
- self.module.get_accessible_gpus_for_seg = Mock(return_value = [2,2,2])
- res = self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_id', 'test_table', 'id_col',
- 'independent_varname', 'output_table', None,
- True, None, None)
- self.assertEqual('response', res.pred_type)
- self.assertEqual(2, res.gpus_per_host)
- self.assertEqual([0,1,2,3,4], res.class_values)
- self.assertEqual(1.0, res.normalizing_const)
- self.assertEqual('text', res.dependent_vartype)
-
- def test_predictbyom_defaults_2(self):
- res = self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_id', 'test_table', 'id_col',
- 'independent_varname', 'output_table',
- self.pred_type, self.use_gpus,
- self.class_values, self.normalizing_const)
- self.assertEqual('prob', res.pred_type)
- self.assertEqual(0, res.gpus_per_host)
- self.assertEqual(['foo', 'bar', 'baaz', 'foo2', 'bar2'], res.class_values)
- self.assertEqual(255.0, res.normalizing_const)
- self.assertEqual('double precision', res.dependent_vartype)
-
- def test_predictbyom_exception_invalid_params(self):
- with self.assertRaises(plpy.PLPYException) as error:
- self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_id', 'test_table', 'id_col',
- 'independent_varname', 'output_table',
- 'invalid_pred_type', self.use_gpus,
- self.class_values, self.normalizing_const)
- self.assertIn('invalid_pred_type', str(error.exception))
-
- with self.assertRaises(plpy.PLPYException) as error:
- self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_id', 'test_table', 'id_col',
- 'independent_varname', 'output_table',
- self.pred_type, self.use_gpus,
- ["foo", "bar", "baaz"], self.normalizing_const)
- self.assertIn('class values', str(error.exception).lower())
-
- with self.assertRaises(plpy.PLPYException) as error:
- self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_id', 'test_table', 'id_col',
- 'independent_varname', 'output_table',
- self.pred_type, self.use_gpus,
- self.class_values, 0)
- self.assertIn('normalizing const', str(error.exception).lower())
-
-
-class MadlibKerasWrapperTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
-
- import madlib_keras_wrapper
- self.subject = madlib_keras_wrapper
-
- self.use_gpus = False
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_get_gpu_memory_fraction(self):
-
- gpus_per_host = 4
- segments_per_host = 4
- result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host)
- self.assertEqual(result, 0.9)
-
- gpus_per_host = 10
- segments_per_host = 4
- result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host)
- self.assertEqual(result, 0.9)
-
- gpus_per_host = 2
- segments_per_host = 6
- result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host)
- self.assertEqual(result, 0.3)
-
- gpus_per_host = 1
- segments_per_host = 4
- result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host)
- self.assertEqual(result, 0.225)
-
- def test_get_device_name_and_set_cuda_env_postgres(self):
- self.subject.is_platform_pg = Mock(return_value = True)
-
- seg_id = -1
- gpus_per_host = 3
-
- self.assertEqual('/gpu:0', self.subject.get_device_name_and_set_cuda_env(
- gpus_per_host, seg_id ))
- self.assertEqual('0,1,2', os.environ['CUDA_VISIBLE_DEVICES'])
-
- gpus_per_host = 0
- self.assertEqual('/cpu:0', self.subject.get_device_name_and_set_cuda_env(
- gpus_per_host, seg_id ))
- self.assertEqual('-1', os.environ['CUDA_VISIBLE_DEVICES'])
-
- def test_get_device_name_and_set_cuda_env_gpdb(self):
- self.subject.is_platform_pg = Mock(return_value = False)
-
- seg_id=3
- gpus_per_host=2
- self.assertEqual('/gpu:0', self.subject.get_device_name_and_set_cuda_env(
- gpus_per_host, seg_id))
- self.assertEqual('1', os.environ['CUDA_VISIBLE_DEVICES'])
-
- gpus_per_host=0
- self.assertEqual('/cpu:0', self.subject.get_device_name_and_set_cuda_env(
- gpus_per_host, seg_id))
- self.assertEqual('-1', os.environ['CUDA_VISIBLE_DEVICES'])
-
-
- def test_split_and_strip(self):
- self.assertEqual(('a','b'), self.subject.split_and_strip(' a = b '))
-
- def test_parse_optimizer(self):
- opt_name = 'SGD'
- final_args = {'lr':0.01, 'decay':1e-6, 'nesterov':True}
- compile_dict = {}
- compile_dict['optimizer']='SGD(lr=0.01, decay=1e-6, nesterov=True)'
- result_name, result_params = self.subject.parse_optimizer(compile_dict)
-
- self.assertEqual(result_name, opt_name)
- self.assertDictEqual(result_params, final_args)
-
- def test_validate_and_literal_eval_keys(self):
- test_dict = {'batch_size':'2', 'epochs':'1', 'verbose':'0'}
- target_dict = {'batch_size':2, 'epochs':1, 'verbose':0}
- literal_eval_fit_params = ['batch_size','epochs','verbose','shuffle',
- 'class_weight','initial_epoch','steps_per_epoch']
- accepted_fit_params = literal_eval_fit_params + ['shuffle']
- result_params = self.subject.validate_and_literal_eval_keys(
- test_dict,
- literal_eval_fit_params,
- accepted_fit_params)
- self.assertDictEqual(result_params, target_dict)
-
- def test_parse_and_validate_fit_params(self):
- result = {'batch_size':2, 'epochs':1, 'verbose':0}
- self.assertDictEqual(result, self.subject.parse_and_validate_fit_params('batch_size=2, epochs=1, verbose=0'))
-
- def test_parse_and_validate_fit_params(self):
- test_str = "batch_size=2, epochs=1, verbose=0"
- fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0}
- result_params = self.subject.parse_and_validate_fit_params(test_str)
- self.assertDictEqual(result_params, fit_dict)
-
- test_str = "batch_size=2, epochs=1, verbose=0, shuffle = 'batch'"
- fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0, 'shuffle':'batch'}
- result_params = self.subject.parse_and_validate_fit_params(test_str)
- self.assertDictEqual(result_params, fit_dict)
-
- test_str = "batch_size=2, epochs=1, verbose=0, shuffle = True"
- fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0, 'shuffle':True}
- result_params = self.subject.parse_and_validate_fit_params(test_str)
- self.assertDictEqual(result_params, fit_dict)
-
- test_str = ""
- fit_dict = {}
- result_params = self.subject.parse_and_validate_fit_params(test_str)
- self.assertDictEqual(result_params, fit_dict)
-
- def test_parse_and_validate_fit_params_duplicate_params(self):
- test_str = "batch_size=1, batch_size=10, verbose=0"
- fit_dict = {'batch_size':10, 'verbose':0}
- result_params = self.subject.parse_and_validate_fit_params(test_str)
- self.assertDictEqual(result_params, fit_dict)
-
- def test_parse_and_validate_fit_params_invalid_fit_param_fail(self):
- test_str = "does_not_exist=2, epochs=1, verbose=0"
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.parse_and_validate_fit_params(test_str)
-
- self.assertIn('not currently accepted', str(error.exception))
-
- test_str = "batch_size=not_lit_eval(1), epochs=1, verbose=0"
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.parse_and_validate_fit_params(test_str)
-
- self.assertIn('invalid input value', str(error.exception))
-
- def test_parse_and_validate_compile_params(self):
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \
- "loss='categorical_crossentropy', metrics=['accuracy']"
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'metrics':['accuracy'], 'loss':'categorical_crossentropy'}
- opt_name,opt_args,result_params = self.subject.parse_and_validate_compile_params(test_str)
- self.assertDictEqual(result_params, compile_dict)
-
- # Test without the metrics
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \
- "loss='categorical_crossentropy'"
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'loss':'categorical_crossentropy'}
- opt_name,opt_args,result_params = self.subject.parse_and_validate_compile_params(test_str)
- self.assertDictEqual(result_params, compile_dict)
-
- def test_validate_metrics_None_pass(self):
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'metrics':['accuracy'], 'loss':'categorical_crossentropy'}
- self.subject._validate_metrics(compile_dict)
-
- def test_validate_metrics_empty_pass(self):
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'metrics':[], 'loss':'categorical_crossentropy'}
- self.subject._validate_metrics(compile_dict)
-
- def test_validate_metrics_two_params_fail(self):
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'metrics':['accuracy','mae'], 'loss':'categorical_crossentropy'}
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject._validate_metrics(compile_dict)
-
- def test_validate_metrics_one_unsupported_fail(self):
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'metrics':['sparse_categorical_accuracy'], 'loss':'categorical_crossentropy'}
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject._validate_metrics(compile_dict)
-
- def test_validate_metrics_two_unsupported_fail(self):
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'metrics':['sparse_categorical_accuracy', 'sparse_categorical_crossentropy'], 'loss':'categorical_crossentropy'}
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject._validate_metrics(compile_dict)
-
- def test_validate_metrics_one_supported_one_unsupported_fail(self):
- compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)',
- 'metrics':['accuracy', 'sparse_categorical_crossentropy'], 'loss':'categorical_crossentropy'}
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject._validate_metrics(compile_dict)
-
- def test_parse_and_validate_compile_params_default_optimizer_pass(self):
- test_str = "optimizer='SGD', loss='categorical_crossentropy'"
- _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str)
- self.assertEqual('SGD', result_dict['optimizer'])
-
- test_str = "optimizer='sgd()', loss='categorical_crossentropy'"
- _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str)
- self.assertEqual('sgd()', result_dict['optimizer'])
-
- def test_parse_and_validate_compile_params_duplicate_loss(self):
- test_str = "optimizer='SGD', loss='loss1', metrics=['accuracy'], loss='loss2'"
- _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str)
- self.assertEqual('loss2', result_dict['loss'])
-
- def test_parse_and_validate_compile_params_no_optimizer_fail(self):
- test_str = "loss='categorical_crossentropy', metrics=['accuracy']"
-
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_compile_params(test_str)
-
- def test_parse_and_validate_compile_params_no_loss_fail(self):
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \
- "metrics=['accuracy']"
-
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_compile_params(test_str)
-
- def test_parse_and_validate_compile_params_dict_metrics_fail(self):
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \
- "loss='categorical_crossentropy', metrics={'0':'accuracy'}"
-
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_compile_params(test_str)
-
- def test_parse_and_validate_compile_params_tensor_loss_weights_fail(self):
-
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \
- " loss='categorical_crossentropy', metrics=['accuracy']," \
- " loss_weights = keras.layers.Input(shape=(32,))"
-
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_compile_params(test_str)
-
- def test_parse_and_validate_compile_params_list_dict_sample_weight_mode_fail(self):
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \
- " loss='categorical_crossentropy', metrics=['accuracy']," \
- " sample_weight_mode = [0,1]"
-
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_compile_params(test_str)
-
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \
- " loss='categorical_crossentropy', metrics=['accuracy']," \
- " sample_weight_mode = {'0':'1'}"
-
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_compile_params(test_str)
-
-
- def test_parse_and_validate_compile_params_target_tensors_fail(self):
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \
- " loss='categorical_crossentropy', metrics=['accuracy']," \
- " target_tensors = keras.layers.Input(shape=(32,))"
-
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_compile_params(test_str)
-
- def test_parse_and_validate_fit_params_callbacks_fail(self):
-
- test_str = "batch_size=2, epochs=1, verbose=0, callbacks=keras.callbacks.Callback()"
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_fit_params(test_str)
-
- def test_parse_and_validate_fit_params_validation_split_fail(self):
-
- test_str = "batch_size=2, epochs=1, verbose=0, validation_split=0.1"
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_fit_params(test_str)
-
- def test_parse_and_validate_fit_params_validation_data_fail(self):
-
- test_str = "batch_size=2, epochs=1, verbose=0, validation_data=(1,2,3)"
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_fit_params(test_str)
-
- def test_parse_and_validate_fit_params_sample_weight_fail(self):
-
- test_str = "batch_size=2, epochs=1, verbose=0, sample_weight=np.array([1,2,3])"
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_fit_params(test_str)
-
- def test_parse_and_validate_fit_params_validation_steps_fail(self):
-
- test_str = "batch_size=2, epochs=1, verbose=0, validation_steps=1"
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_fit_params(test_str)
-
- def test_parse_and_validate_fit_params_validation_freq_fail(self):
- test_str = "batch_size=2, epochs=1, verbose=0, validation_freq=1"
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_fit_params(test_str)
-
- test_str = "batch_size=2, epochs=1, verbose=0, validation_freq=[1]"
- with self.assertRaises(plpy.PLPYException):
- self.subject.parse_and_validate_fit_params(test_str)
-
- def test_parse_and_validate_compile_params_syntax_error_fail(self):
- #missing end parentheses
- test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True"
- with self.assertRaises(ValueError) as error:
- self.subject.parse_and_validate_compile_params(test_str)
- self.assertIn('could not convert string to float', str(error.exception))
-
- #missing beginning parentheses
- test_str = "optimizer=SGDlr=0.01, decay=1e-6, nesterov=True)," \
- " loss='categorical_crossentropy'"
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.parse_and_validate_compile_params(test_str)
- self.assertIn('not currently accepted', str(error.exception))
-
- #missing comma
- test_str = "optimizer=SGD(lr=0.01 decay=1e-6, nesterov=True)," \
- " loss='categorical_crossentropy'"
- with self.assertRaises(ValueError) as error:
- self.subject.parse_and_validate_compile_params(test_str)
- self.assertIn('invalid literal for float', str(error.exception))
-
- def test_parse_and_validate_fit_params_invalid_optimizer_fail(self):
- test_str = "optimizer='SGD1', loss='categorical_crossentropy'"
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.parse_and_validate_compile_params(test_str)
- self.assertIn('invalid optimizer', str(error.exception))
-
-
-class MadlibKerasFitCommonValidatorTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras_validator
- self.subject = madlib_keras_validator
-
- def tearDown(self):
- self.module_patcher.stop()
-
-
- def test_is_valid_metrics_compute_frequency_True_None(self):
- self.subject.FitCommonValidator._validate_common_args = Mock()
- obj = self.subject.FitCommonValidator(
- 'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, None, False, False, [0],
- 'module_name')
- self.assertEqual(True, obj._is_valid_metrics_compute_frequency())
-
- def test_is_valid_metrics_compute_frequency_True_num(self):
- self.subject.FitCommonValidator._validate_common_args = Mock()
- obj = self.subject.FitCommonValidator(
- 'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, 3, False, False, [0],
- 'module_name')
- self.assertEqual(True, obj._is_valid_metrics_compute_frequency())
-
- def test_is_valid_metrics_compute_frequency_False_zero(self):
- self.subject.FitCommonValidator._validate_common_args = Mock()
- obj = self.subject.FitCommonValidator(
- 'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, 0, False, False, [0],
- 'module_name')
- self.assertEqual(False, obj._is_valid_metrics_compute_frequency())
-
- def test_is_valid_metrics_compute_frequency_False_greater(self):
- self.subject.FitCommonValidator._validate_common_args = Mock()
- obj = self.subject.FitCommonValidator(
- 'test_table', 'val_table', 'model_table', 'model_arch_table', 2,
- 'dep_varname', 'independent_varname', 5, 6, False, False, [0],
- 'module_name')
- self.assertEqual(False, obj._is_valid_metrics_compute_frequency())
-
-
-class InputValidatorTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras_validator
- self.module = madlib_keras_validator
- self.subject = self.module.InputValidator
-
- self.module_name = 'module'
- self.test_table = 'test_table'
- self.model_table = 'model_table'
- self.id_col = 'id_col'
- self.ind_var = 'ind_var'
- self.model_arch_table = 'model_arch_table'
- self.model_id = 2
- self.num_classes = 1598
- self.model = Sequential()
- self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu',
- input_shape=(1,1,1,), padding='same'))
- self.model.add(Dense(self.num_classes))
- self.classes = ['train', 'boat', 'car', 'airplane']
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_validate_pred_type_invalid_pred_type(self):
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.validate_pred_type(
- self.module_name, 'invalid_pred_type', ['cat', 'dog'])
- self.assertIn('type', str(error.exception).lower())
-
- def test_validate_class_values_greater_than_1600_class_values(self):
- self.model.add(Dense(1599))
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.validate_class_values(
- self.module_name, range(1599), 'prob', self.model.to_json())
- self.assertIn('1600', str(error.exception))
-
- def test_validate_class_values_valid_class_values_prob(self):
- self.subject.validate_class_values(
- self.module_name, range(self.num_classes), 'prob', self.model.to_json())
- self.subject.validate_class_values(
- self.module_name, None, 'prob', self.model.to_json())
-
- def test_validate_class_values_valid_pred_type_valid_class_values_response(self):
- self.subject.validate_class_values(
- self.module_name, range(self.num_classes), 'response', self.model.to_json())
- self.subject.validate_class_values(
- self.module_name, None, 'response', self.model.to_json())
-
- def test_validate_input_shape_shapes_do_not_match(self):
- # minibatched data
- self.plpy_mock_execute.return_value = [{'shape': [1,3,32,32]}]
- with self.assertRaises(plpy.PLPYException):
- self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 2, True)
- # non-minibatched data
- self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 32,'n_2': 32,'n_3': 3}]
- with self.assertRaises(plpy.PLPYException):
- self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 1)
- self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 3}]
- with self.assertRaises(plpy.PLPYException):
- self.subject.validate_input_shape(
- self.test_table, self.ind_var, [3,32], 1)
-
- def test_validate_input_shape_shapes_match(self):
- # minibatched data
- self.plpy_mock_execute.return_value = [{'shape': [1,32,32,3]}]
- self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 2, True)
- # non-minibatched data
- self.plpy_mock_execute.return_value = [{'n_0': 32,'n_1': 32,'n_2': 3}]
- self.subject.validate_input_shape(
- self.test_table, self.ind_var, [32,32,3], 1)
-
- def test_validate_model_arch_table_none_values(self):
- with self.assertRaises(plpy.PLPYException) as error:
- obj = self.subject.validate_model_arch_table(
- self.module_name, None, self.model_id)
- self.assertIn('null', str(error.exception).lower())
-
- self.module.input_tbl_valid = Mock()
- with self.assertRaises(plpy.PLPYException) as error:
- obj = self.subject.validate_model_arch_table(
- self.module_name, self.model_arch_table, None)
- self.assertIn('id', str(error.exception).lower())
-
- def test_validate_gpu_config_with_gpu_all_segments(self):
-
- self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': 'all_segments'}]
- obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1])
-
- def test_validate_gpu_config_no_gpu_all_segments(self):
-
- self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': 'all_segments'}]
- with self.assertRaises(plpy.PLPYException) as error:
- obj = self.subject._validate_gpu_config(self.module_name, 'foo', [0])
- self.assertIn('missing gpus', str(error.exception).lower())
-
- def test_validate_gpu_config_with_gpu_valid_seg_list(self):
-
- self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': [0,1]}]
- obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1,1,0,1])
-
- def test_validate_gpu_config_with_gpu_invalid_seg_list(self):
-
- self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': [0,1]}]
- with self.assertRaises(plpy.PLPYException) as error:
- obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1,0,0,1])
- self.assertIn('does not have gpu', str(error.exception).lower())
-
-class MadlibSerializerTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras_serializer
- self.subject = madlib_keras_serializer
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_deserialize_image_1d_weights_null_state_returns_none(self):
- self.assertEqual(None, self.subject.deserialize_as_image_1d_weights(None))
-
- def test_deserialize_image_1d_weights_returns_not_none(self):
- dummy_state = np.array([2,3,4,5,6], dtype=np.float32)
- res = self.subject.deserialize_as_image_1d_weights(dummy_state.tostring())
- self.assertEqual(2, res[0])
- self.assertEqual([3,4,5,6], res[1].tolist())
-
- def test_deserialize_nd_weights_null_input_returns_none(self):
- dummy_state = np.array([0,1,2,3,4,5,6], dtype=np.float32)
- self.assertEqual(None, self.subject.deserialize_as_nd_weights(dummy_state.tostring(), None))
- self.assertEqual(None, self.subject.deserialize_as_nd_weights(None, [1, 2, 3]))
- self.assertEqual(None, self.subject.deserialize_as_nd_weights(None, None))
-
- def test_deserialize_nd_weights_valid_input_returns_not_none(self):
- dummy_model_weights = np.array([3,4,5], dtype=np.float32)
- dummy_model_shape = [(2, 1, 1, 1), (1,)]
- res = self.subject.deserialize_as_nd_weights(dummy_model_weights.tostring(),
- dummy_model_shape)
- self.assertEqual([[[[3.0]]], [[[4.0]]]], res[0].tolist())
- self.assertEqual([5], res[1].tolist())
-
- def test_deserialize_nd_weights_invalid_input_fails(self):
- # pass an invalid state with missing model weights
- invalid_model_weights = np.array([1,2], dtype=np.float32)
- dummy_model_shape = [(2, 1, 1, 1), (1,)]
-
- # we expect raise exception because we cannot reshape
- # model weights of size 0 into shape (2,2,3,1)
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(),
- dummy_model_shape)
-
- invalid_model_weights = np.array([1,2,3,4], dtype=np.float32)
- dummy_model_shape = [(2, 2, 3, 1), (1,)]
- # we expect raise exception because we cannot reshape
- # model weights of size 2 into shape (2,2,3,1)
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(),
- dummy_model_shape)
-
- invalid_model_weights = np.array([0,1,2,3,4], dtype=np.float32)
- dummy_model_shape = [(2, 1), (1,)]
- # we expect to raise exception because we cannot reshape
- # model weights of size 2 into shape (1,)
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(),
- dummy_model_shape)
-
- def test_serialize_image_nd_weights_none_weights_returns_none(self):
- res = self.subject.serialize_state_with_nd_weights(0, None)
- self.assertEqual(None , res)
-
- def test_serialize_image_nd_weights_valid_output(self):
- res = self.subject.serialize_state_with_nd_weights(0, [np.array([1, 3]),
- np.array([4,5])])
- self.assertEqual(np.array([0,1,3,4,5], dtype=np.float32).tostring(),
- res)
-
- def test_serialize_image_1d_weights_none_weights_returns_none(self):
- res = self.subject.serialize_state_with_1d_weights(0, None)
- self.assertEqual(None , res)
-
- def test_serialize_image_1d_weights_valid_output(self):
- res = self.subject.serialize_state_with_1d_weights(0, np.array([1, 3, 4, 5]))
- self.assertEqual(np.array([0,1,3,4,5], dtype=np.float32).tostring(),
- res)
-
-
-class MadlibKerasHelperTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras_helper
- self.subject = madlib_keras_helper
- self.input_data = [32, 32, 3]
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_expand_input_dims(self):
- self.assertEqual(np.array(self.input_data).shape, (3,))
- res = self.subject.expand_input_dims(self.input_data)
- self.assertEqual(res.shape, (1, 3))
-
- def test_strip_trailing_nulls_from_class_values(self):
- self.assertEqual(['cat', 'dog'],
- self.subject.strip_trailing_nulls_from_class_values(
- ['cat', 'dog']))
- self.assertEqual([None, 'cat', 'dog'],
- self.subject.strip_trailing_nulls_from_class_values(
- [None, 'cat', 'dog']))
- self.assertEqual([None, 'cat', 'dog'],
- self.subject.strip_trailing_nulls_from_class_values(
- [None, 'cat', 'dog', None, None]))
- self.assertEqual(['cat', 'dog'],
- self.subject.strip_trailing_nulls_from_class_values(
- ['cat', 'dog', None, None]))
- self.assertEqual([None],
- self.subject.strip_trailing_nulls_from_class_values(
- [None, None]))
-
- def test_get_gpus_per_one_seg_gpu_gpdb(self):
-
- self.subject.is_platform_pg = Mock(return_value = False)
-
- self.plpy_mock_execute.side_effect = \
- [ [],
- [ {'hostname': 'mdw0', 'count' : 1}],
- [],
- [ {'hostname': 'mdw0', 'segment_id' : 0},
- {'hostname': 'mdw1', 'segment_id' : 1},
- {'hostname': 'mdw2', 'segment_id' : 2}
- ]]
-
- self.assertEqual([1,0,0], self.subject.get_accessible_gpus_for_seg(
- 'schema_madlib', 2, 'foo'))
-
- def test_get_gpus_per_mult_seg_gpu_gpdb(self):
-
- self.subject.is_platform_pg = Mock(return_value = False)
-
- self.plpy_mock_execute.side_effect = \
- [[],
- [ {'hostname': 'mdw0', 'count' : 1}],
- [],
- [ {'hostname': 'mdw0', 'segment_id' : 0},
- {'hostname': 'mdw0', 'segment_id' : 1},
- {'hostname': 'mdw1', 'segment_id' : 2},
- {'hostname': 'mdw1', 'segment_id' : 3}
- ]]
-
- self.assertEqual([1,1,0,0], self.subject.get_accessible_gpus_for_seg(
- 'schema_madlib', 2, 'foo'))
-
- def test_get_gpus_per_no_gpu_gpdb(self):
-
- self.subject.is_platform_pg = Mock(return_value = False)
-
- self.plpy_mock_execute.side_effect = [[],[],[]]
- with self.assertRaises(plpy.PLPYException) as error:
- self.subject.get_accessible_gpus_for_seg('schema_madlib', 2, 'foo')
- self.assertIn('no gpus configured on hosts', str(error.exception).lower())
-
-class MadlibKerasEvaluationTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras
- self.subject = madlib_keras
-
- self.model = Sequential()
- self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu',
- input_shape=(1,1,1,), padding='same'))
- self.model.add(Flatten())
-
- self.compile_params = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']"
- self.model_weights = [3,4,5,6]
- self.serialized_weights = np.array(self.model_weights, dtype='float32'
- ).tostring()
- self.loss = 0.5947071313858032
- self.accuracy = 1.0
- self.dist_key_mapping = [0,1,2]
- self.accessible_gpus_for_seg = [0]
-
- #self.model.evaluate = Mock(return_value = [self.loss, self.accuracy])
-
- self.independent_var_real = [[[[0.5]]]] * 10
- self.dependent_var_int = [[0,1]] * 10
-
- # Params as bytea
- self.independent_var = np.array(self.independent_var_real, dtype=np.float32).tobytes()
- self.dependent_var = np.array(self.dependent_var_int, dtype=np.int16).tobytes()
-
- self.independent_var_shape = [10,1,1,1]
- self.dependent_var_shape = [10,2]
- # We test on segment 0, which has 3 buffers filled with 10 identical
- # images each, or 30 images total
- self.total_images_per_seg = [3*len(self.dependent_var_int),20,40]
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def _test_internal_keras_eval_transition_first_buffer(self, is_platform_pg):
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
- self.subject.is_platform_pg = Mock(return_value = is_platform_pg)
- starting_image_count = 0
- ending_image_count = len(self.dependent_var_int)
-
- # last iteration call
-
- k = {'SD' : {}}
- state = [0,0,0]
- new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(),
- self.serialized_weights, self.compile_params, 0,
- self.dist_key_mapping, 0, 4,
- self.total_images_per_seg, False, self.accessible_gpus_for_seg, True, **k)
-
- agg_loss, agg_accuracy, image_count = new_state
-
- self.assertEqual(ending_image_count, image_count)
- # Call set_session once for gpdb (but not for postgres)
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # loss and accuracy should be unchanged
- self.assertAlmostEqual(self.loss * image_count, agg_loss, 4)
- self.assertAlmostEqual(self.accuracy * image_count, agg_accuracy, 4)
- # Clear session and sess.close must not get called for the first buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
- self.assertTrue(k['SD']['segment_model'])
-
- # Non-final call
-
- self.subject.K.set_session.reset_mock()
- self.subject.K.clear_session.reset_mock()
- k = {'SD' : {}}
- state = [0,0,0]
-
- new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(),
- self.serialized_weights, self.compile_params, 0,
- self.dist_key_mapping, 0, 4,
- self.total_images_per_seg, False, self.accessible_gpus_for_seg, False, **k)
- agg_loss, agg_accuracy, image_count = new_state
-
- self.assertEqual(ending_image_count, image_count)
- # set_session must not get called for the first buffer
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # loss and accuracy should be unchanged
- self.assertAlmostEqual(self.loss * image_count, agg_loss, 4)
- self.assertAlmostEqual(self.accuracy * image_count, agg_accuracy, 4)
- # Clear session and sess.close must not get called for the first buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
- self.assertTrue(k['SD']['segment_model'])
-
- def _test_internal_keras_eval_transition_middle_buffer(self, is_platform_pg):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
- self.subject.is_platform_pg = Mock(return_value = is_platform_pg)
-
- starting_image_count = len(self.dependent_var_int)
- ending_image_count = starting_image_count + len(self.dependent_var_int)
-
- # last iteration call
-
- k = {'SD' : {}}
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
-
- state = [self.loss * starting_image_count, self.accuracy * starting_image_count, starting_image_count]
- k['SD']['segment_model'] = self.model
- k['SD']['sess'] = Mock()
-
- new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(),
- 'dummy_model_weights', None, 0,
- self.dist_key_mapping, 0, 4,
- self.total_images_per_seg, False, self.accessible_gpus_for_seg, True, **k)
-
- agg_loss, agg_accuracy, image_count = new_state
-
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # loss and accuracy should be unchanged
- self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4)
- self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4)
- # Clear session and sess.close must not get called for the middle buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
-
- # Non-last iteration call
-
- self.subject.K.set_session.reset_mock()
- self.subject.K.clear_session.reset_mock()
- k = {'SD' : {}}
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
-
- state = [self.loss * starting_image_count, self.accuracy * starting_image_count, starting_image_count]
- k['SD']['segment_model'] = self.model
- k['SD']['sess'] = Mock()
-
- new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(),
- 'dummy_model_weights', None, 0,
- self.dist_key_mapping, 0, 4,
- self.total_images_per_seg, False, self.accessible_gpus_for_seg, False, **k)
- agg_loss, agg_accuracy, image_count = new_state
-
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # loss and accuracy should be unchanged
- self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4)
- self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4)
- # Clear session and sess.close must not get called for the middle buffer
- self.assertEqual(0, self.subject.K.clear_session.call_count)
-
- def _test_internal_keras_eval_transition_last_buffer(self, is_platform_pg):
- #TODO should we mock tensorflow's close_session and keras'
- # clear_session instead of mocking the function `K.clear_session`
- self.subject.K.set_session = Mock()
- self.subject.K.clear_session = Mock()
- self.subject.is_platform_pg = Mock(return_value = is_platform_pg)
-
- starting_image_count = 2*len(self.dependent_var_int)
- ending_image_count = starting_image_count + len(self.dependent_var_int)
-
- k = {'SD' : {}}
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
-
- state = [self.loss * starting_image_count,
- self.accuracy * starting_image_count,
- starting_image_count]
-
- k['SD']['segment_model'] = self.model
- k['SD']['sess'] = Mock()
-
- new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(),
- 'dummy_model_weights', None, 0,
- self.dist_key_mapping, 0, 4,
- self.total_images_per_seg, False, self.accessible_gpus_for_seg, True, **k)
- agg_loss, agg_accuracy, image_count = new_state
-
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # loss and accuracy should be unchanged
- self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4)
- self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4)
- # Clear session and sess.close must get called for the last buffer in gpdb,
- # but not in postgres
- self.assertEqual(1, self.subject.K.clear_session.call_count)
-
- # Non-final call
-
- self.subject.K.set_session.reset_mock()
- self.subject.K.clear_session.reset_mock()
- k = {'SD' : {}}
-
- self.subject.compile_and_set_weights(self.model, self.compile_params,
- '/cpu:0', self.serialized_weights)
-
- state = [self.loss * starting_image_count,
- self.accuracy * starting_image_count,
- starting_image_count]
-
- k['SD']['segment_model'] = self.model
- k['SD']['sess'] = Mock()
-
- new_state = self.subject.internal_keras_eval_transition(
- state, self.dependent_var , self.independent_var,
- self.dependent_var_shape, self.independent_var_shape,
- self.model.to_json(),
- 'dummy_model_weights', None, 0,
- self.dist_key_mapping, 0, 4,
- self.total_images_per_seg, False, self.accessible_gpus_for_seg, False, **k)
-
- agg_loss, agg_accuracy, image_count = new_state
-
- self.assertEqual(ending_image_count, image_count)
- # set_session is always called
- self.assertEqual(1, self.subject.K.set_session.call_count)
- # loss and accuracy should be unchanged
- self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4)
- self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4)
- # Clear session and sess.close must not get called in non-final iterations
- self.assertEqual(0, self.subject.K.clear_session.call_count)
-
- def test_internal_keras_eval_transition_first_buffer_pg(self):
- self._test_internal_keras_eval_transition_first_buffer(True)
-
- def test_internal_keras_eval_transition_first_buffer_gpdb(self):
- self._test_internal_keras_eval_transition_first_buffer(False)
-
- def test_internal_keras_eval_transition_middle_buffer_pg(self):
- self._test_internal_keras_eval_transition_middle_buffer(True)
-
- def test_internal_keras_eval_transition_middle_buffer_gpdb(self):
- self._test_internal_keras_eval_transition_middle_buffer(False)
-
- def test_internal_keras_eval_transition_last_buffer_pg(self):
- self._test_internal_keras_eval_transition_last_buffer(True)
-
- def test_internal_keras_eval_transition_last_buffer_gpdb(self):
- self._test_internal_keras_eval_transition_last_buffer(False)
-
- def test_internal_keras_eval_merge(self):
- image_count = self.total_images_per_seg[0]
- state1 = [3.0*self.loss, 3.0*self.accuracy, image_count]
- state1 = state1
- state2 = [2.0*self.loss, 2.0*self.accuracy, image_count+30]
- state2 = state2
- merged_state = self.subject.internal_keras_eval_merge(state1,state2)
- agg_loss = merged_state[0]
- agg_accuracy = merged_state[1]
- image_count_total = merged_state[2]
-
- self.assertEqual( 2*image_count+30 , image_count_total )
- self.assertAlmostEqual( 5.0*self.loss, agg_loss, 2)
- self.assertAlmostEqual( 5.0*self.accuracy, agg_accuracy, 2)
-
- def test_internal_keras_eval_merge_none_first(self):
- image_count = self.total_images_per_seg[0]
- input_state = [self.loss, self.accuracy, image_count]
- merged_state = self.subject.internal_keras_eval_merge(None, input_state)
- agg_loss = merged_state[0]
- agg_accuracy = merged_state[1]
- image_count_total = merged_state[2]
-
- self.assertEqual(image_count, image_count_total)
- self.assertAlmostEqual(self.loss, agg_loss, 2)
- self.assertAlmostEqual(self.accuracy, agg_accuracy, 2)
-
- def test_internal_keras_eval_merge_none_second(self):
- image_count = self.total_images_per_seg[0]
- input_state = [self.loss, self.accuracy, image_count]
- merged_state = self.subject.internal_keras_eval_merge(input_state, None)
- agg_loss = merged_state[0]
- agg_accuracy = merged_state[1]
- image_count_total = merged_state[2]
-
- self.assertEqual(image_count, image_count_total)
- self.assertAlmostEqual(self.loss, agg_loss, 2)
- self.assertAlmostEqual(self.accuracy, agg_accuracy, 2)
-
- def test_internal_keras_eval_merge_both_none(self):
- result = self.subject.internal_keras_eval_merge(None,None)
- self.assertEqual(None, result)
-
- def test_internal_keras_eval_final(self):
- image_count = self.total_images_per_seg[0]
- input_state = [image_count*self.loss, image_count*self.accuracy, image_count]
-
- output_state = self.subject.internal_keras_eval_final(input_state)
- self.assertEqual(len(output_state), 2)
- agg_loss = output_state[0]
- agg_accuracy = output_state[1]
-
- self.assertAlmostEqual(self.loss, agg_loss,2)
- self.assertAlmostEqual(self.accuracy, agg_accuracy,2)
-
- def internal_keras_eval_final_none(self):
- result = self.subject.internal_keras_eval_final(None)
- self.assertEqual(result, None)
-
- def test_internal_keras_eval_final_image_count_zero(self):
- input_state = [0, 0, 0]
-
- with self.assertRaises(plpy.PLPYException):
- result = self.subject.internal_keras_eval_final(input_state)
-
- def tearDown(self):
- self.module_patcher.stop()
-
-if __name__ == '__main__':
- unittest.main()
-# ---------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_gpu_info.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_gpu_info.py_in
deleted file mode 100644
index cc885f9..0000000
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_gpu_info.py_in
+++ /dev/null
@@ -1,88 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-m4_changequote(`<!', `!>')
-
-import sys
-import numpy as np
-import os
-from os import path
-# Add dl module to the pythonpath.
-sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))))
-sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))
-
-import unittest
-from mock import *
-import plpy_mock as plpy
-
-
-
-class GPUInfoTestCase(unittest.TestCase):
- def setUp(self):
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import madlib_keras_gpu_info
- self.module = madlib_keras_gpu_info
- self.module.output_tbl_valid = Mock()
- self.table_name = 'does_not_matter'
-
- def tearDown(self):
- self.module_patcher.stop()
-
- def test_gpu_configuration_invalid_source(self):
- with self.assertRaises(plpy.PLPYException):
- self.module.gpu_configuration('schema', self.table_name, 'invalid_source')
-
- def test_gpu_configuration_none_source(self):
- self.module.gpu_configuration('schema', self.table_name, None)
- self.assertIn('tensorflow', str(self.plpy_mock_execute.call_args_list))
- self.assertNotIn('nvidia', str(self.plpy_mock_execute.call_args_list))
-
- def test_gpu_configuration_tensorflow(self):
- self.module.gpu_configuration('schema', self.table_name, 'tensorflow')
- self.assertIn('tensorflow', str(self.plpy_mock_execute.call_args_list))
- self.assertNotIn('nvidia', str(self.plpy_mock_execute.call_args_list))
-
- def test_gpu_configuration_nvidia(self):
- self.module.gpu_configuration('schema', self.table_name, 'nvidia')
- self.assertIn('nvidia', str(self.plpy_mock_execute.call_args_list))
- self.assertNotIn('tensorflow', str(self.plpy_mock_execute.call_args_list))
-
- def test_gpu_configuration_tensorflow_uppercase(self):
- self.module.gpu_configuration('schema', self.table_name, 'TENSORFLOW')
- self.assertIn('tensorflow', str(self.plpy_mock_execute.call_args_list))
- self.assertNotIn('nvidia', str(self.plpy_mock_execute.call_args_list))
-
- def test_gpu_configuration_nvidia_mixedcase(self):
- self.module.gpu_configuration('schema', self.table_name, 'NVidIa')
- self.assertIn('nvidia', str(self.plpy_mock_execute.call_args_list))
- self.assertNotIn('tensorflow', str(self.plpy_mock_execute.call_args_list))
-
-
-if __name__ == '__main__':
- unittest.main()
-# ---------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
deleted file mode 100644
index 57e08a5..0000000
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
+++ /dev/null
@@ -1,140 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-m4_changequote(`<!', `!>')
-
-import sys
-import os
-from os import path
-# Add convex module to the pythonpath.
-sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))))
-sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))
-
-import keras
-import unittest
-from mock import *
-import plpy_mock as plpy
-
-class LoadModelSelectionTableTestCase(unittest.TestCase):
- def setUp(self):
- # The side effects of this class(writing to the output table) are not
- # tested here. They are tested in dev-check.
- self.plpy_mock = Mock(spec='error')
- patches = {
- 'plpy': plpy
- }
-
- self.plpy_mock_execute = MagicMock()
- plpy.execute = self.plpy_mock_execute
-
- self.module_patcher = patch.dict('sys.modules', patches)
- self.module_patcher.start()
- import deep_learning.madlib_keras_model_selection
- self.module = deep_learning.madlib_keras_model_selection
- self.module.MstLoaderInputValidator._validate_input_args = \
- MagicMock()
-
- self.subject = self.module.MstLoader
- self.model_selection_table = 'mst_table'
- self.model_arch_table = 'model_arch_library'
- self.model_id_list = [1]
- self.compile_params_list = [
- """
- loss='categorical_crossentropy',
- optimizer='Adam(lr=0.1)',
- metrics=['accuracy']
- """,
- """
- loss='categorical_crossentropy',
- optimizer='Adam(lr=0.01)',
- metrics=['accuracy']
- """,
- """
- loss='categorical_crossentropy',
- optimizer='Adam(lr=0.001)',
- metrics=['accuracy']
- """
- ]
- self.fit_params_list = [
- "batch_size=5,epochs=1",
- "batch_size=10,epochs=1"
- ]
-
- def test_mst_table_dimension(self):
- generate_mst = self.subject(
- self.model_selection_table,
- self.model_arch_table,
- self.model_id_list,
- self.compile_params_list,
- self.fit_params_list
- )
- self.assertEqual(6, len(generate_mst.msts))
-
- def test_invalid_input_args(self):
- self.module.MstLoaderInputValidator \
- ._validate_input_args \
- .side_effect = plpy.PLPYException('Invalid input args')
- with self.assertRaises(plpy.PLPYException):
- generate_mst = self.subject(
- self.model_selection_table,
- self.model_arch_table,
- self.model_id_list,
- self.compile_params_list,
- self.fit_params_list
- )
-
- def test_duplicate_params(self):
- self.model_id_list = [1, 1, 2]
- self.compile_params_list = [
- """
- loss='categorical_crossentropy',
- optimizer='Adam(lr=0.1)',
- metrics=['accuracy']
- """,
- """
- loss='categorical_crossentropy',
- optimizer='Adam(lr=0.1)',
- metrics=['accuracy']
- """,
- """
- loss='categorical_crossentropy',
- optimizer='Adam(lr=0.001)',
- metrics=['accuracy']
- """
- ]
- self.fit_params_list = [
- "batch_size= 5,epochs=1",
- "epochs=1 ,batch_size=5",
- "batch_size=10,epochs =1"
- ]
- generate_mst = self.subject(
- self.model_selection_table,
- self.model_arch_table,
- self.model_id_list,
- self.compile_params_list,
- self.fit_params_list
- )
- self.assertEqual(8, len(generate_mst.msts))
-
- def tearDown(self):
- self.module_patcher.stop()
-
-if __name__ == '__main__':
- unittest.main()
-# ---------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net.py_in b/src/ports/postgres/modules/elastic_net/elastic_net.py_in
index c6dec53..43c05e1 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net.py_in
@@ -282,7 +282,7 @@ def elastic_net_help(schema_madlib, family_or_optimizer=None, **kwargs):
def elastic_net_train(schema_madlib, source_table, model_table, dependent_varname,
independent_varname, regress_family, alpha, lambda_value,
standardize, grouping_col, optimizer,
- optimizer_params, excluded, max_iter, tolerance,
+ optimizer_params, exclude, max_iter, tolerance,
**kwargs):
"""
A wrapper for all variants of elastic net regularization.
@@ -350,7 +350,7 @@ def _internal_elastic_net_train(
schema_madlib, source_table, model_table, dependent_varname,
independent_varname, grouping_col,
regress_family, alpha, lambda_value,
- standardize, optimizer, optimizer_params, excluded,
+ standardize, optimizer, optimizer_params, exclude,
max_iter, tolerance, **kwargs):
tbl_summary = add_postfix(model_table, "_summary")
@@ -358,7 +358,7 @@ def _internal_elastic_net_train(
# handle all special cases of independent_varname
independent_varname, outstr_array = analyze_input_str(
schema_madlib, source_table,
- independent_varname, dependent_varname, excluded)
+ independent_varname, dependent_varname, exclude)
# get the grouping info
reserved_cols =['regress_family', 'coef_all',
@@ -495,7 +495,7 @@ def _check_args(tbl_source, col_ind_var, col_dep_var):
def analyze_input_str(schema_madlib, tbl_source,
- col_ind_var, col_dep_var, excluded):
+ col_ind_var, col_dep_var, exclude):
"""
Make input strings and output strings compatible with functions
@@ -511,7 +511,7 @@ def analyze_input_str(schema_madlib, tbl_source,
col_types_dict = dict(get_cols_and_types(tbl_source))
cols = col_types_dict.keys()
- s = _string_to_array(excluded) if excluded is not None else []
+ s = _string_to_array(exclude) if exclude is not None else []
for each_col in cols:
if each_col not in s and each_col != col_dep_var:
@@ -537,7 +537,7 @@ def analyze_input_str(schema_madlib, tbl_source,
if columns_exist_in_table(tbl_source, [col_ind_var], schema_madlib):
# if the input is a column name and not an expression
return analyze_single_input_str(schema_madlib, tbl_source,
- col_ind_var, excluded)
+ col_ind_var, exclude)
else:
# if input is an expression resulting in an array output
matched = is_string_formatted_as_array_expression(col_ind_var)
@@ -562,7 +562,7 @@ def analyze_input_str(schema_madlib, tbl_source,
def analyze_single_input_str(schema_madlib, tbl_source, col_ind_var,
- excluded=None):
+ exclude=None):
"""
Args:
@param schema_madlib: string, Name of schema where MADlib is installed
@@ -588,8 +588,8 @@ def analyze_single_input_str(schema_madlib, tbl_source, col_ind_var,
""".format(tbl_source=tbl_source,
col_ind_var=col_ind_var))[0]["dimension"]
- if excluded is not None:
- s = _string_to_array(excluded)
+ if exclude is not None:
+ s = _string_to_array(exclude)
invalid_excluded = """
Elastic Net error: When the independent variable is
an array column, excluded values can only be indices
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net.sql_in b/src/ports/postgres/modules/elastic_net/elastic_net.sql_in
index c1aaebf..2358b4e 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net.sql_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net.sql_in
@@ -895,7 +895,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train (
grouping_col TEXT,
optimizer TEXT,
optimizer_params TEXT,
- excluded TEXT,
+ exclude TEXT,
max_iter INTEGER,
tolerance DOUBLE PRECISION
) RETURNS VOID AS $$
@@ -905,7 +905,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train (
schema_madlib, tbl_source, tbl_result, col_dep_var,
col_ind_var, regress_family, alpha, lambda_value,
standardize, grouping_col, optimizer, optimizer_params,
- excluded, max_iter, tolerance)
+ exclude, max_iter, tolerance)
$$ LANGUAGE plpythonu
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -923,7 +923,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train (
grouping_columns TEXT,
optimizer TEXT,
optimizer_params TEXT,
- excluded TEXT,
+ exclude TEXT,
max_iter INTEGER
) RETURNS VOID AS $$
BEGIN
@@ -945,7 +945,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train (
grouping_columns TEXT,
optimizer TEXT,
optimizer_params TEXT,
- excluded TEXT
+ exclude TEXT
) RETURNS VOID AS $$
BEGIN
PERFORM MADLIB_SCHEMA.elastic_net_train($1, $2, $3, $4, $5, $6, $7, $8,
@@ -1157,7 +1157,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_gaussian_predict (
ind_var DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', '__elastic_net_gaussian_predict'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1176,7 +1176,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_binomial_predict (
ind_var DOUBLE PRECISION[]
) RETURNS BOOLEAN AS
'MODULE_PATHNAME', '__elastic_net_binomial_predict'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1195,7 +1195,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_binomial_prob (
ind_var DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', '__elastic_net_binomial_prob'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1207,7 +1207,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__elastic_net_binomial_loglikelihood (
ind_var DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', '__elastic_net_binomial_loglikelihood'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1240,7 +1240,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_transition (
step_decay DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'gaussian_igd_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1250,7 +1250,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_merge (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'gaussian_igd_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1259,7 +1259,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_final (
state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'gaussian_igd_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
@@ -1337,7 +1337,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_state_diff (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', '__gaussian_igd_state_diff'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1349,7 +1349,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_result (
tolerance DOUBLE PRECISION
) RETURNS MADLIB_SCHEMA.__elastic_net_result AS
'MODULE_PATHNAME', '__gaussian_igd_result'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1372,7 +1372,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_transition (
random_stepsize INTEGER
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'gaussian_fista_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1382,7 +1382,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_merge (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'gaussian_fista_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1391,7 +1391,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_final (
state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'gaussian_fista_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
@@ -1439,7 +1439,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_state_diff (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', '__gaussian_fista_state_diff'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1448,7 +1448,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_result (
in_state DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__elastic_net_result AS
'MODULE_PATHNAME', '__gaussian_fista_result'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1472,7 +1472,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_transition (
step_decay DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'binomial_igd_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1482,7 +1482,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_merge (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'binomial_igd_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1491,7 +1491,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_final (
state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'binomial_igd_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
@@ -1569,7 +1569,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_state_diff (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', '__binomial_igd_state_diff'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1581,7 +1581,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_result (
tolerance DOUBLE PRECISION
) RETURNS MADLIB_SCHEMA.__elastic_net_result AS
'MODULE_PATHNAME', '__binomial_igd_result'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1604,7 +1604,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_transition (
random_stepsize INTEGER
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'binomial_fista_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1614,7 +1614,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_merge (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'binomial_fista_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1623,7 +1623,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_final (
state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'binomial_fista_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
@@ -1671,7 +1671,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_state_diff (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', '__binomial_fista_state_diff'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--
@@ -1680,7 +1680,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_result (
in_state DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__elastic_net_result AS
'MODULE_PATHNAME', '__binomial_fista_result'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/deep_learning/__init__.py_in b/src/ports/postgres/modules/facebook_prophet/__init__.py_in
similarity index 100%
rename from src/ports/postgres/modules/deep_learning/__init__.py_in
rename to src/ports/postgres/modules/facebook_prophet/__init__.py_in
diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_alg.py_in b/src/ports/postgres/modules/facebook_prophet/prophet_alg.py_in
new file mode 100644
index 0000000..a1ea25e
--- /dev/null
+++ b/src/ports/postgres/modules/facebook_prophet/prophet_alg.py_in
@@ -0,0 +1,349 @@
+# -*- coding:utf-8 -*-
+# Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+#
+# openGauss is licensed under Mulan PSL v2.
+# You can use this software according to the terms
+# and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+# http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------
+# Timeseries predict use facebook module prophet.
+# ----------------------------------------------------------------------
+import plpy
+import cPickle as pickle
+
+from utilities.validate_args import table_exists
+from utilities.validate_args import table_is_empty
+from utilities.validate_args import columns_exist_in_table
+from utilities.validate_args import get_expr_type
+from utilities.validate_args import get_cols
+from utilities.utilities import _assert
+from utilities.control import MinWarning
+
+
+def prophet_fit(schema_madlib, input_table, model_table, column_map, params_str, verbose, **kwargs):
+ """
+ Args:
+ @param input_table, -- name of input table
+ @param model_table, -- name of output table
+ @param column_map, -- map relation of column
+ @param params_str, -- names of paraters for prophet
+ @param verbose -- verbose or not
+ """
+ with MinWarning('warning'):
+ _validate_args(schema_madlib, input_table, model_table, params_str)
+
+ sql = """
+ DROP TABLE IF EXISTS {model_table};
+ CREATE TABLE {model_table} m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `')
+ AS
+ (
+ SELECT
+ 1 as id,
+ (mdl_results).y_type,
+ (mdl_results).model,
+ (mdl_results).params
+ FROM (
+ SELECT
+ {schema_madlib}.__get_fbmodel_as_bytea__(
+ '{input_table}',
+ $${column_map}$$,
+ $${params_str}$$,
+ {verbose}
+ ) as mdl_results
+ )q
+ );
+ """.format(
+ schema_madlib=schema_madlib,
+ model_table=model_table,
+ input_table=input_table,
+ column_map=column_map,
+ params_str=params_str,
+ verbose=verbose
+ )
+
+ if verbose:
+ plpy.info(sql)
+ plpy.execute(sql)
+
+ return "Prophet_fit over and model is saved in {model_table}".format(model_table=model_table)
+
+
+# ------------------------------------------------------------------------------
+# -- prophet predict function ------------------------------------------------------------
+# ------------------------------------------------------------------------------
+def prophet_predict(schema_madlib, model_table, output_table, periods, freq, cap, floor, include_history, verbose, **kwargs):
+ """
+ Args:
+ @param model_table, -- name of model table
+ @param output_table, -- name of output table
+ @param periods, -- value of predict length
+ @param freq, -- frequency of predict value
+ @param cap, -- max value of predict value(logistic)
+ @param floor, -- min value of predict value(logistic)
+ @param inlcude_history
+ """
+ with MinWarning('warning'):
+ _validate_predict(schema_madlib, model_table, output_table, periods)
+
+ sql = """select y_type, model from {model_table};""".format(model_table=model_table)
+ if verbose:
+ plpy.info(sql)
+ result = plpy.execute(sql)
+ y_type = result[0]['y_type']
+
+ model = result[0]['model']
+ model = pickle.loads(model)
+
+ if freq:
+ future = model.make_future_dataframe(periods=periods, freq=freq, include_history=include_history)
+ else:
+ future = model.make_future_dataframe(periods=periods, include_history=include_history)
+ if cap is not None:
+ future['cap'] = cap
+ if floor is not None:
+ future['floor'] = floor
+ forecast = model.predict(future)
+
+ columns = forecast.columns
+ column_list, value_list = [], []
+ for column in columns:
+ if column == 'ds':
+ column_list.append('{column} date'.format(column=column))
+ value_list.append("'{{{column}}}'".format(column=column))
+ else:
+ column_list.append('{column} {y_type}'.format(column=column, y_type=y_type))
+ value_list.append('{{{column}}}'.format(column=column))
+ column_information = ', '.join(column_list)
+ value_information = ', '.join(value_list)
+ plpy.info("column_information is: " + str(column_information))
+ plpy.info("value_information is: " + str(value_information))
+
+ sql = """
+ DROP TABLE IF EXISTS {output_table};
+ CREATE TABLE {output_table} ({column_information})
+ """.format(
+ output_table=output_table,
+ column_information=column_information
+ )
+ if verbose:
+ plpy.info(sql)
+ plpy.execute(sql)
+
+
+ sql = """INSERT INTO {output_table} VALUES ({value_information});""".format(output_table=output_table,
+ value_information=value_information)
+ plpy.info("insert sql is {sql}".format(sql=sql))
+ predict_length = len(forecast['ds'])
+ for i in range(predict_length):
+ plpy.execute(
+ sql.format(**{col:forecast[col][i] for col in columns})
+ )
+
+ return "prophet predict over and result is saved in {output_table}".format(output_table=output_table)
+
+
+# ------------------------------------------------------------------------------
+# -- validate args ------------------------------------------------------------
+# ------------------------------------------------------------------------------
+def _validate_args(schema_madlib, input_table, model_table, params_str):
+ """
+ @brief validate the arguments
+ """
+ # 1, check input_table: name.
+ _assert(input_table and input_table.strip().lower() not in ('null', ''),
+ "Prophet error: Invalid data table name!")
+ _assert(table_exists(input_table),
+ "Prophet error: Data table does not exist!")
+ _assert(not table_is_empty(input_table),
+ "Prophet error: Data table is empty!")
+
+ # 3, check output table.
+ _assert(model_table is not None and model_table.strip().lower() not in ('null', ''),
+ "Prophet error: Invalid output table name!")
+ _assert(not table_exists(model_table, only_first_schema=True),
+ "Output table name already exists. Drop the table before calling the function.")
+
+ # 4, check params_str
+ if params_str != '':
+ # let prophet to check the parameter
+ pass
+
+
+def _validate_predict(schema_madlib, model_table, output_table, periods):
+ """
+ @brief validate the predict arguments
+ """
+ # 1, check input_table: name.
+ _assert(model_table and model_table.strip().lower() not in ('null', ''),
+ "Prophet error: Invalid model table name!")
+ _assert(table_exists(model_table),
+ "Prophet error: model table does not exist!")
+ _assert(not table_is_empty(model_table),
+ "Prophet error: Data table is empty!")
+
+ # 3, check output table.
+ _assert(output_table is not None and output_table.strip().lower() not in ('null', ''),
+ "Prophet error: Invalid output_table table name!")
+ _assert(not table_exists(output_table, only_first_schema=True),
+ "Output table name already exists. Drop the table before calling the function.")
+
+ # 4, check periods
+ _assert(isinstance(periods, int) and periods > 0, "Prophet error: Invalid parameter 'periods'")
+
+
+# ------------------------------------------------------------------------------
+# -- Online help function ------------------------------------------------------
+# ------------------------------------------------------------------------------
+def prophet_fit_help_message(schema_madlib, message, **kwargs):
+ """ Help message for Prophet_fit
+
+ @brief
+ Args:
+ @param schema_madlib string, Name of the schema madlib
+ @param message string, Help message indicator
+
+ Returns:
+ String. Contain the help message string
+ """
+ if not message:
+ help_string="""
+ -----------------------------------------------------------------------
+ SUMMARY
+ -----------------------------------------------------------------------
+ facebook prophet API timeseries predict.
+
+ For more details on function usage:
+ SELECT {schema_madlib}.prophet_fit('usage')
+ """
+ elif message in ['usage', 'help', '?']:
+ help_string="""
+ -----------------------------------------------------------------------
+ USAGE
+ -----------------------------------------------------------------------
+ SELECT {schema_madlib}.prophet_fit(
+ input_table TEXT, -- in, Data table name
+ model_table TEXT, -- out, save model and features
+ column_map TEXT, -- map relation of column : 'ds', 'y', 'floor', 'cap'
+ params_str TEXT, -- prophet parameters
+ verbose BOOLEAN -- Boolean, whether to print more info, default is False
+ );
+
+ -----------------------------------------------------------------------
+ parameters
+ -----------------------------------------------------------------------
+ below is optional parameters:
+
+ column_map:
+ Prophet has specified column name as follows:
+ 'ds': column of date
+ 'y': column of predict value
+ 'cap': upper limit of predict value
+ 'floor': lower limit of predict value
+
+ so in order to simplify the usage of prophet, we allow user to provide a map relationship between
+ specified column name and corresponding column name, for example:
+ date | value
+ ------------------
+ 2000-1-1 | 100
+ 2000-1-2 | 120
+ 2000-1-3 | 130
+
+ in this case, we should provide a map like {{'ds': 'date', 'y': 'value'}}
+ if user do not provide map relationship, then use the default {{'ds': 'ds', 'y': 'y'}}, if you want to use 'cap' and 'floor',
+ you should definite it in column_map, otherwise will not use.
+
+ params_str:
+ For example, if the input is:
+ $${{'growth': 'linear', 'changepoints': ['2000-01-01', '2000-02-02'], 'yearly_seasonality': True}}$$
+
+ a full input query:
+ select {schema_madlib}.prophet_fit('inputTable', 'output_modelTable', $${{'ds':'ds', 'y': 'y', 'cap': 'cap'}}$$,
+ $${{'growth': 'linear', 'changepoints': ['2000-01-01', '2000-02-02'], 'yearly_seasonality': True, 'holidays': ['playoffs']}}$$, True);
+
+ -----------------------------------------------------------------------
+ OUTPUT
+ -----------------------------------------------------------------------
+ The output table ('model_table' above) has the following columns:
+ id -- index
+ model -- binary model
+
+ note: the type of ds_column must be DATE
+
+ """
+ else:
+ help_string="No such option. Use {schema_madlib}.prophet_fit()"
+
+ return help_string.format(schema_madlib=schema_madlib)
+
+
+def prophet_predict_help_message(schema_madlib, message, **kwargs):
+ """ Help message for Prophet_predict
+
+ @brief
+ Args:
+ @param schema_madlib string, Name of the schema madlib
+ @param message string, Help message indicator
+
+ Returns:
+ String. Contain the help message string
+ """
+ if not message:
+ help_string="""
+ -----------------------------------------------------------------------
+ SUMMARY
+ -----------------------------------------------------------------------
+ facebook prophet API timeseries predict.
+
+ For more details on function usage:
+ SELECT {schema_madlib}.prophet_predict('usage')
+ """
+ elif message in ['usage', 'help', '?']:
+ help_string="""
+ -----------------------------------------------------------------------
+ USAGE
+ -----------------------------------------------------------------------
+ SELECT {schema_madlib}.prophet_predict(
+ model_table TEXT, -- in, model table name
+ output_table TEXT, -- out, save predict result
+ periods INTEGER, -- the number of periods to forecast forward.
+ freq TEXT, -- 'day', 'week', 'month', 'quarter', 'year', 1(1 sec), 60(1 minute) or 3600(1 hour)
+ cap DOUBLE PRECISION, -- the max value in future predict(logistic mode)
+ floor DOUBLE PRECISION, -- the min value in future predict(logistic mode)
+ include_history BOOLEAN, -- to include the historical dates in the data frame for predictions.
+ verbose BOOLEAN -- whether to print more info, default is False
+ );
+
+ -----------------------------------------------------------------------
+ parameters
+ -----------------------------------------------------------------------
+ below is optional parameters:
+
+ periods:
+ periods >= 1
+ cap, floor:
+ if growth is 'logistic', user should provide 'cap' or 'floor', if do not use it, just set NULL.
+
+ example: select {schema_madlib}.prophet_predict('modelTable', 'output_Table', 1000, '10T', 10, NULL, False, False);
+ example: select {schema_madlib}.prophet_predict('modelTable', 'output_Table', 1000, '10T', NULL, NULL, False, False);
+
+
+ -----------------------------------------------------------------------
+ OUTPUT
+ -----------------------------------------------------------------------
+ The output table include all result of prophet_predict
+ """
+ else:
+ help_string="No such option. Use {schema_madlib}.prophet_fit()"
+
+ return help_string.format(schema_madlib=schema_madlib)
+
diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_alg.sql_in b/src/ports/postgres/modules/facebook_prophet/prophet_alg.sql_in
new file mode 100644
index 0000000..9bace08
--- /dev/null
+++ b/src/ports/postgres/modules/facebook_prophet/prophet_alg.sql_in
@@ -0,0 +1,186 @@
+/*
+ * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+ *
+ * openGauss is licensed under Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *
+ * http://license.coscl.org.cn/MulanPSL2
+ *
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * -------------------------------------------------------------------------
+ */
+
+-------------------------------------------------------------------------------------------------------------------------------
+-- Build facebook Prophet models in-database - -
+-------------------------------------------------------------------------------------------------------------------------------
+--
+-----------
+-- Note: --
+-----------
+-- 1) Porphet uses facebook-API. However we're limited by the maximum
+-- field-size in Greenplum/Postgres which is currently 1 GB.
+-- 2) If your dataset is much larger (> 1 GB), it is strongly recommended that you use MADlib's models so that
+-- training & scoring will happen in-parallel on all nodes.
+-------------------------------------------------------------------------------------------------------------------------------
+
+m4_include(`SQLCommon.m4')
+
+----------------------------------------------------------------------------------------------
+------ Help messages
+----------------------------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit(
+ message TEXT
+) RETURNS TEXT AS $XX$
+ PythonFunction(facebook_prophet, prophet_alg, prophet_fit_help_message)
+$XX$ LANGUAGE plpythonu IMMUTABLE;
+-------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit()
+RETURNS TEXT AS $XX$
+ SELECT MADLIB_SCHEMA.prophet_fit(''::TEXT);
+$XX$
+LANGUAGE sql IMMUTABLE;
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict(
+ message TEXT
+) RETURNS TEXT AS $XX$
+ PythonFunction(facebook_prophet, prophet_alg, prophet_predict_help_message)
+$XX$ LANGUAGE plpythonu IMMUTABLE;
+-------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict()
+RETURNS TEXT AS $XX$
+ SELECT MADLIB_SCHEMA.prophet_predict(''::TEXT);
+$XX$
+LANGUAGE sql IMMUTABLE;
+
+
+----------------------------------------------------------------------------------------------
+------ prophet fit
+----------------------------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit(
+ input_table TEXT, -- in, dataset table
+ model_table TEXT, -- out, save model and features
+ column_map TEXT, -- map relation of column {'ds' 'y' 'floor' 'cap'}
+ params_str TEXT, -- prophet parameters
+ verbose BOOLEAN
+)
+RETURNS TEXT AS $$
+PythonFunction(facebook_prophet, prophet_alg, prophet_fit)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit(
+ input_table TEXT,
+ model_table TEXT,
+ column_map TEXT,
+ params_str TEXT
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_fit($1, $2, $3, $4, False);
+$$ LANGUAGE sql;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit(
+ input_table TEXT,
+ model_table TEXT,
+ column_map TEXT
+
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_fit($1, $2, $3, NULL);
+$$ LANGUAGE sql;
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit(
+ input_table TEXT,
+ model_table TEXT
+
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_fit($1, $2, NULL);
+$$ LANGUAGE sql;
+
+
+--def prophet_predict(schema_madlib, model_table, output_table, periods, verbose, **kwargs):
+
+
+----------------------------------------------------------------------------------------------
+------ prophet predict
+----------------------------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict(
+ model_table TEXT, -- in, dataset table
+ output_table TEXT, -- out, save model and features
+ periods INTEGER, -- predict length
+ freq TEXT, -- frequency of input_dataset
+ cap DOUBLE PRECISION, -- max value of predict value(logistic)
+ floor DOUBLE PRECISION, -- min value of predict value(logistic)
+ include_history BOOLEAN, -- include history dataset in output_table
+ verbose BOOLEAN -- log or not
+)
+RETURNS TEXT AS $$
+PythonFunction(facebook_prophet, prophet_alg, prophet_predict)
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict(
+ model_table TEXT, -- in, dataset table
+ output_table TEXT, -- out, save model and features
+ periods INTEGER, -- predict length
+ freq TEXT, -- frequency of input_dataset
+ cap DOUBLE PRECISION, -- max value of predict value(logistic)
+ floor DOUBLE PRECISION, -- min value of predict value(logistic)
+ include_history BOOLEAN -- include history dataset in output_table
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, $5, $6, $7, False);
+$$ LANGUAGE sql;
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict(
+ model_table TEXT, -- in, dataset table
+ output_table TEXT, -- out, save model and features
+ periods INTEGER, -- predict length
+ freq TEXT, -- frequency of input_dataset
+ cap DOUBLE PRECISION, -- max value of predict value(logistic)
+ floor DOUBLE PRECISION -- min value of predict value(logistic)
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, $5, $6, False);
+$$ LANGUAGE sql;
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict(
+ model_table TEXT, -- in, dataset table
+ output_table TEXT, -- out, save model and features
+ periods INTEGER, -- predict length
+ freq TEXT, -- frequency of input_dataset
+ cap DOUBLE PRECISION -- max value of predict value(logistic)
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, $5, NULL);
+$$ LANGUAGE sql;
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict(
+ model_table TEXT, -- in, dataset table
+ output_table TEXT, -- out, save model and features
+ periods INTEGER, -- predict length
+ freq TEXT -- frequency of input_dataset
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, NULL);
+$$ LANGUAGE sql;
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict(
+ model_table TEXT, -- in, dataset table
+ output_table TEXT, -- out, save model and features
+ periods INTEGER -- predict length
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, NULL);
+$$ LANGUAGE sql;
+
+
diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_utilities.py_in b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.py_in
new file mode 100644
index 0000000..2eaba82
--- /dev/null
+++ b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.py_in
@@ -0,0 +1,107 @@
+# -*- coding:utf-8 -*-
+# Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+#
+# openGauss is licensed under Mulan PSL v2.
+# You can use this software according to the terms
+# and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+# http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ----------------------------------------------------------------------------
+
+import re
+import ast
+import plpy
+
+import pandas as pd
+import cPickle as pickle
+from fbprophet import Prophet
+
+from utilities.validate_args import get_expr_type
+
+
+support_type_list = ('TINYINT', 'SMALLINT', 'INTEGER', 'BINARY_INTEGER', 'BIGINT',
+ 'NUMERIC', 'DECIMAL',
+ 'SMALLSERIAL', 'SERIAL', 'BIGSERIAL',
+ 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'FLOAT8', 'FLOAT', 'BINARY_DOUBLE', 'DEC',
+ 'BOOLEAN')
+
+
+def get_fbmodel(schema_madlib, input_table, column_map, params_str, verbose, **kwargs):
+ """
+ train prophet model and return as bytea
+ Args:
+ @param input_table, -- name of input table
+ @param column_map, -- map relationship of specified name of column and current name
+ @param params_str, -- paramater of prophet fit function
+ @param verbose -- display log
+ """
+ if params_str is None or params_str is "" or ":" not in params_str:
+ params_str = "{}"
+ plpy.info("prophet_fit() params_str type is: " + params_str)
+
+ if column_map is None or column_map is "" or ":" not in column_map:
+ column_map = "{'ds': 'ds', 'y': 'y'}"
+ plpy.info("Prophet_fit() column_map relation is: " + column_map)
+
+ params = ast.literal_eval(re.sub("[\\t]", "", params_str).strip())
+ column_map_dict = ast.literal_eval(re.sub("[\\t]", "", column_map).strip())
+
+ y_type = check_input_type(schema_madlib, input_table, column_map_dict)
+
+ if 'holidays' in params.keys():
+ holidays = pd.DataFrame()
+ holiday_tables = params['holidays']
+ sql = """select * from {holiday_table}"""
+ for holiday_table in holiday_tables:
+ sql = sql.format(holiday_table=holiday_table)
+ if verbose:
+ plpy.info(sql)
+ result = plpy.execute(sql)
+ holiday_df = pd.DataFrame.from_records(result)
+ holidays = pd.concat((holidays, holiday_df))
+ params['holidays'] = holidays
+
+ real_columns = ', '.join([value for value in column_map_dict.values()])
+ sql = """select {real_columns} from {input_table};""".format(input_table=input_table, real_columns=real_columns)
+ if verbose:
+ plpy.info(sql)
+
+ result = plpy.execute(sql)
+ df = pd.DataFrame.from_records(result)
+
+ # change column name to accord with prophet requirement
+ for s_name, c_name in column_map_dict.items():
+ if s_name != c_name:
+ df.rename(columns={c_name: s_name}, inplace=True)
+ model = Prophet(**params)
+ model.fit(df)
+
+ return (y_type, pickle.dumps(model), str(params))
+
+
+# ------------------------------------------------------------------------------
+# -- validate column type of input table ------------------------------------------------------------
+# ------------------------------------------------------------------------------
+def check_input_type(schema_madlib, input_table, column_map_dict):
+ """
+ func: check the column type of input_table
+ """
+ for s_col, c_col in column_map_dict.items():
+ col_type = get_expr_type(c_col, input_table)
+ if s_col == 'ds':
+ if 'time' not in col_type.lower() and 'date' not in col_type.lower():
+ plpy.error("prophet Error: the type of {column} must be DATE".format(column=c_col))
+ else:
+ if col_type.upper() not in support_type_list:
+ plpy.error("prophet Error: the type of {column} is not supported".format(column=c_col))
+
+ y_type = get_expr_type(column_map_dict['y'], input_table)
+
+ return y_type
diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_utilities.sql_in b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.sql_in
new file mode 100644
index 0000000..4a132b4
--- /dev/null
+++ b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.sql_in
@@ -0,0 +1,54 @@
+/*
+ * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+ *
+ * openGauss is licensed under Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *
+ * http://license.coscl.org.cn/MulanPSL2
+ *
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * -------------------------------------------------------------------------
+ */
+
+-------------------------------------------------------------------------------------------------------------------------------
+-- Build facebook Prophet models in-database - -
+-------------------------------------------------------------------------------------------------------------------------------
+--
+-----------
+-- Note: --
+-----------
+-- 1) Porphet uses facebook-API. However we're limited by the maximum
+-- field-size in Greenplum/Postgres which is currently 1 GB.
+-- 2) If your dataset is much larger (> 1 GB), it is strongly recommended that you use MADlib's models so that
+-- training & scoring will happen in-parallel on all nodes.
+-------------------------------------------------------------------------------------------------------------------------------
+
+m4_include(`SQLCommon.m4')
+
+
+------------------------------------------------
+-- Type for internal train results
+------------------------------------------------
+DROP TYPE IF EXISTS MADLIB_SCHEMA.prophet_fit_results_type cascade;
+CREATE TYPE MADLIB_SCHEMA.prophet_fit_results_type AS(
+ y_type TEXT, -- save X
+ model bytea, -- dump model
+ params TEXT -- parameters used
+);
+
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__get_fbmodel_as_bytea__(
+ input_table TEXT, -- dataset table
+ column_map TEXT, -- map relation of column {'ds' 'y' 'floor' 'cap'}
+ params_str TEXT, -- xgboost parameters
+ verbose BOOLEAN
+)
+RETURNS MADLIB_SCHEMA.prophet_fit_results_type AS $$
+PythonFunction(facebook_prophet, prophet_utilities, get_fbmodel)
+$$ LANGUAGE plpythonu;
+
diff --git a/src/ports/postgres/modules/facebook_prophet/test/prophet_test.sql_in b/src/ports/postgres/modules/facebook_prophet/test/prophet_test.sql_in
new file mode 100644
index 0000000..37b8c88
--- /dev/null
+++ b/src/ports/postgres/modules/facebook_prophet/test/prophet_test.sql_in
@@ -0,0 +1,1522 @@
+/*
+* Test case of facebook-prophet
+*/
+
+m4_include(`SQLCommon.m4')
+
+select MADLIB_SCHEMA.prophet_fit();
+select MADLIB_SCHEMA.prophet_fit('usage');
+
+DROP TABLE IF EXISTS ts_data;
+CREATE TABLE ts_data (date date, value float);
+
+
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:20:00', 5.592186415908717);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:30:00', 5.213219014779573);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:40:00', 5.349837267365468);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:50:00', 5.348009861982742);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:00:00', 5.141486962759949);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:10:00', 5.831837644472339);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:20:00', 5.586104738236943);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:30:00', 5.424038616617589);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:40:00', 5.3953055294925);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:50:00', 5.08062402180196);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:00:00', 5.216144429037251);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:10:00', 5.89720541594696);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:20:00', 5.903379069573093);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:30:00', 5.108443063954224);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:40:00', 5.829479550836606);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:50:00', 5.983691151745764);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:00:00', 5.911012559550056);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:10:00', 5.267628873207424);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:20:00', 5.440281847906313);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:30:00', 5.056752106326658);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:40:00', 5.57695918188404);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:50:00', 5.708959203703136);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:00:00', 5.837530980321919);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:10:00', 5.397656757637807);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:20:00', 5.80272939233592);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:30:00', 5.05404686767896);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:40:00', 5.599526207554924);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:50:00', 5.624649182696436);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:00:00', 5.630874726351715);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:10:00', 5.885412123046953);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:20:00', 5.7281711367203965);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:30:00', 5.367035247234945);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:40:00', 5.647728469736866);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:50:00', 5.422412855958562);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:00:00', 5.084810422982988);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:10:00', 5.045434995298759);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:20:00', 5.903595563281863);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:30:00', 5.8464706036749226);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:40:00', 5.423780748843331);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:50:00', 5.698599390020942);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:00:00', 5.620828657830448);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:10:00', 5.853793611596745);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:20:00', 5.1160341760917465);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:30:00', 5.760091737386397);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:40:00', 5.547488495151962);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:50:00', 5.074259195264116);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:00:00', 5.767149486416109);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:10:00', 5.5412954545856525);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:20:00', 5.671416922868077);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:30:00', 5.214281328444059);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:40:00', 5.704061227110594);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:50:00', 5.981904654103126);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:00:00', 5.811672256762379);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:10:00', 5.636629064171528);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:20:00', 5.222889294671177);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:30:00', 5.7944970498250745);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:40:00', 5.301092523899824);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:50:00', 5.378620727476704);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:00:00', 5.832205512624812);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:10:00', 5.1940020098503785);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:20:00', 5.2801063135872415);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:30:00', 5.333104829752228);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:40:00', 5.845933760530428);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:50:00', 5.932520038392692);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:00:00', 5.64679296474034);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:10:00', 5.209244473298587);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:20:00', 5.379575393446499);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:30:00', 5.551810996707294);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:40:00', 5.98125647954102);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:50:00', 5.352159243180884);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:00:00', 5.98297448186513);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:10:00', 5.05840496515969);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:20:00', 5.071467132599093);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:30:00', 5.508013094801815);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:40:00', 5.593629750578956);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:50:00', 5.0073554230453405);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:00:00', 5.08721378732741);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:10:00', 5.659885556354787);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:20:00', 5.422996896584455);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:30:00', 5.430291377769443);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:40:00', 5.661941353506331);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:50:00', 5.222650073421543);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:00:00', 5.422560019488827);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:10:00', 5.2976478671030405);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:20:00', 5.584338761217304);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:30:00', 5.028576011172816);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:40:00', 5.247034856187782);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:50:00', 5.229344731201686);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:00:00', 5.471182531178972);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:10:00', 5.0949254922245055);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:20:00', 5.683636089958879);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:30:00', 5.437006581310596);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:40:00', 5.245413304985197);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:50:00', 5.522111051119028);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:00:00', 5.005599900215963);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:10:00', 5.53702445367031);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:20:00', 5.5766246582094405);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:30:00', 5.258581727612666);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:40:00', 5.464358328063363);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:50:00', 5.91011805664996);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:00:00', 10.88382409798199);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:10:00', 10.555427090136963);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:20:00', 10.2695546171855);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:30:00', 10.96434493248081);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:40:00', 10.005061878030595);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:50:00', 10.053893326397173);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:00:00', 10.207155270283303);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:10:00', 10.217068622058886);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:20:00', 10.2800096297076);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:30:00', 10.084258019727397);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:40:00', 10.870604507956386);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:50:00', 10.139519698694448);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:00:00', 10.966851433652065);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:10:00', 10.208452839253603);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:20:00', 10.677742584132282);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:30:00', 10.20991792799135);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:40:00', 10.243176260100649);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:50:00', 10.24725814854844);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:00:00', 10.76947018545776);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:10:00', 10.637158701289039);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:20:00', 10.531066872723857);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:30:00', 10.699064458359702);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:40:00', 10.857099725561811);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:50:00', 10.947585602968706);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:00:00', 10.072388396154338);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:10:00', 10.269329386566012);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:20:00', 10.100058130861127);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:30:00', 10.630305282558199);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:40:00', 10.276992479817986);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:50:00', 10.823962547747168);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:00:00', 10.938196297447686);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:10:00', 10.751352407863825);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:20:00', 10.55267731888166);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:30:00', 10.95764805135116);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:40:00', 10.319600419308607);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:50:00', 10.894597987994707);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:00:00', 10.561011949231316);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:10:00', 10.558376344084758);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:20:00', 10.54238813342208);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:30:00', 10.353533152566689);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:40:00', 10.94670826781977);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:50:00', 10.867785908808944);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:00:00', 10.743813978598242);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:10:00', 10.553624735869224);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:20:00', 10.48483757931147);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:30:00', 10.842613332127966);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:40:00', 10.381878265543788);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:50:00', 10.044634020757982);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:00:00', 10.606953617509358);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:10:00', 10.63043787557256);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:20:00', 10.558000303473458);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:30:00', 10.119618312591788);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:40:00', 10.420324078784883);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:50:00', 10.831853130351744);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:00:00', 10.383033393248413);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:10:00', 10.630842365846464);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:20:00', 10.076453778579774);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:30:00', 10.228368345858518);
+INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:40:00', 10.636636225999974);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:50:00', 10.809387112125291);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:00:00', 10.555836272306514);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:10:00', 10.17361788915921);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:20:00', 10.935490280962807);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:30:00', 10.336876569080902);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:40:00', 10.315564948016368);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:50:00', 10.0503251516345);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:00:00', 10.686498994079804);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:10:00', 10.488270765930583);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:20:00', 10.382238780237266);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:30:00', 10.664282558119293);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:40:00', 10.482673868507927);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:50:00', 10.690901645112259);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:00:00', 10.808960869817462);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:10:00', 10.892103395445046);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:20:00', 10.889652181902528);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:30:00', 10.087235381673107);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:40:00', 10.354527252749081);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:50:00', 10.704626831711339);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:00:00', 10.653290677771649);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:10:00', 10.527049427857113);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:20:00', 10.193508167768368);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:30:00', 10.192935509184334);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:40:00', 10.807842757555397);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:50:00', 10.224608561871785);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:00:00', 10.926938435835226);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:10:00', 10.432633335671081);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:20:00', 10.640811656424226);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:30:00', 10.99419293027347);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:40:00', 10.397638983480087);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:50:00', 10.284742522103349);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:00:00', 10.74937007324194);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:10:00', 10.643810271386382);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:20:00', 10.937587864934304);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:30:00', 10.996094208628847);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:40:00', 10.933699502883464);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:50:00', 10.009530363086641);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:00:00', 10.753814613452027);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:10:00', 10.044290103015088);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:20:00', 10.112825899390046);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:30:00', 10.466639947711943);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:40:00', 15.453979216734295);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:50:00', 15.733941145738592);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:00:00', 15.101733596739034);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:10:00', 15.649935609313715);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:20:00', 15.541164284639525);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:30:00', 15.42338136917619);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:40:00', 15.731067751045787);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:50:00', 15.580468045277195);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:00:00', 15.26256456459248);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:10:00', 15.644521316946905);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:20:00', 15.118407201560444);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:30:00', 15.635310837394789);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:40:00', 15.914518468164172);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:50:00', 15.987868390442582);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:00:00', 15.42341614964402);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:10:00', 15.034129189040149);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:20:00', 15.025541704618867);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:30:00', 15.418416247429313);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:40:00', 15.861698475422092);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:50:00', 15.62331366505197);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:00:00', 15.720846440190945);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:10:00', 15.445679787641966);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:20:00', 15.159921163629136);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:30:00', 15.239201767432013);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:40:00', 15.787583155910015);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:50:00', 15.368527703049724);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:00:00', 15.966288763031649);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:10:00', 15.932848769581149);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:20:00', 15.12096784657616);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:30:00', 15.044036821254055);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:40:00', 15.031949588422053);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:50:00', 15.360696548582007);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:00:00', 15.548304448740831);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:10:00', 15.321778753166079);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:20:00', 15.957690239835765);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:30:00', 15.135722181550765);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:40:00', 15.480346629237996);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:50:00', 15.968521145540633);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:00:00', 15.747070467680564);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:10:00', 15.894977279535198);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:20:00', 15.43725890131943);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:30:00', 15.29963684376817);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:40:00', 15.949624102437316);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:50:00', 15.78710097386125);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:00:00', 15.383243943979766);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:10:00', 15.928153538177432);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:20:00', 15.301232004415052);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:30:00', 15.993251015437984);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:40:00', 15.508285598925116);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:50:00', 15.013755434803087);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:00:00', 15.075152055184526);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:10:00', 15.984582015476505);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:20:00', 15.790632765623062);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:30:00', 15.934347484557644);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:40:00', 15.444429332540322);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:50:00', 15.086377880107932);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:00:00', 15.811316282743125);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:10:00', 15.178644553055447);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:20:00', 15.411280626267777);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:30:00', 15.847018277776037);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:40:00', 15.784696373701456);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:50:00', 15.128075902863381);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:00:00', 15.344736721321144);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:10:00', 15.61790225217054);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:20:00', 15.274615030013528);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:30:00', 15.808473672968695);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:40:00', 15.020708990567766);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:50:00', 15.30059613604699);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:00:00', 15.498659236830926);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:10:00', 15.387232492871144);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:20:00', 15.323700631336765);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:30:00', 15.806051242162896);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:40:00', 15.247407375461128);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:50:00', 15.901856510495984);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:00:00', 15.439721502108116);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:10:00', 15.289082346518994);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:20:00', 15.059525497475034);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:30:00', 15.270817224386326);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:40:00', 15.871151199428525);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:50:00', 15.282120255097865);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:00:00', 15.316159525858552);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:10:00', 15.152434228608037);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:20:00', 15.040599454382349);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:30:00', 15.327120128690266);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:40:00', 15.113808122667509);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:50:00', 15.108852157293637);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:00:00', 15.601574694793966);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:10:00', 15.765348827942226);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:20:00', 15.44357776208238);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:30:00', 15.21025279747494);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:40:00', 15.998788041370798);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:50:00', 15.531252123355971);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:00:00', 15.400264196778496);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:10:00', 15.659934846250227);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:20:00', 15.254682422239783);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:30:00', 15.881353351807892);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:40:00', 15.943747331476715);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:50:00', 15.234881305321348);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:00:00', 15.830517028216903);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:10:00', 15.93265269144619);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:20:00', 20.852791006475595);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:30:00', 20.35657131996931);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:40:00', 20.32344079327797);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:50:00', 20.09582415408);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:00:00', 20.733996132413107);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:10:00', 20.68778370171752);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:20:00', 20.467058725004406);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:30:00', 20.379565806425727);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:40:00', 20.42779269193495);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:50:00', 20.094034979346922);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:00:00', 20.227793103797048);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:10:00', 20.936411620696283);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:20:00', 20.619209447538687);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:30:00', 20.035610633780962);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:40:00', 20.445424163695513);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:50:00', 20.206654001756675);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:00:00', 20.55722279943204);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:10:00', 20.41050994653755);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:20:00', 20.549783559583858);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:30:00', 20.446097907452973);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:40:00', 20.48449618191396);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:50:00', 20.747795682422506);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:00:00', 20.16175127009515);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:10:00', 20.186956399208864);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:20:00', 20.109471316760608);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:30:00', 20.501471852177858);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:40:00', 20.46868658303998);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:50:00', 20.039331198435775);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:00:00', 20.733510062349858);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:10:00', 20.77518818818196);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:20:00', 20.593513060310155);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:30:00', 20.37800130713625);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:40:00', 20.51535234903095);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:50:00', 20.799764794377783);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:00:00', 20.05562466198948);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:10:00', 20.918634517305108);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:20:00', 20.46738766522704);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:30:00', 20.410966096668435);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:40:00', 20.20109304019048);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:50:00', 20.425794443629037);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:00:00', 20.764648965476564);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:10:00', 20.73638437971804);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:20:00', 20.86784935806068);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:30:00', 20.106516840699392);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:40:00', 20.26814388324929);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:50:00', 20.15582088751805);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:00:00', 20.365411463552753);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:10:00', 20.68406685337937);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:20:00', 20.43711892282266);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:30:00', 20.80948770424489);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:40:00', 20.76846628710522);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:50:00', 20.116343590304385);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:00:00', 20.31241560439532);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:10:00', 20.69080441002456);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:20:00', 20.90764764555535);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:30:00', 20.975810293040016);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:40:00', 20.17280945563313);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:50:00', 20.538743400676477);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:00:00', 20.237998746198095);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:10:00', 20.79471167783884);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:20:00', 20.938577199072107);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:30:00', 20.501043185915197);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:40:00', 20.87298186827212);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:50:00', 20.70039747550453);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:00:00', 20.330010498371948);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:10:00', 20.965259301979472);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:20:00', 20.463913248270238);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:30:00', 20.335555892985514);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:40:00', 20.225981568350765);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:50:00', 20.328455432029735);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:00:00', 20.409318013480828);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:10:00', 20.433148262692203);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:20:00', 20.96312507182427);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:30:00', 20.27135526020552);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:40:00', 20.25937747505829);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:50:00', 20.86249812853205);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:00:00', 20.766973613848872);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:10:00', 20.922823389181865);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:20:00', 20.696072941144376);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:30:00', 20.742725299116763);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:40:00', 20.00019076674507);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:50:00', 20.066985912811532);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:00:00', 20.66343326593504);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:10:00', 20.969064689272464);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:20:00', 20.62983584739398);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:30:00', 20.2379671764757);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:40:00', 20.579703611822953);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:50:00', 20.915224922908354);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:00:00', 20.05043609646298);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:10:00', 20.360897759018673);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:20:00', 20.90506269170392);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:30:00', 20.4103630269194);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:40:00', 20.287250690598615);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:50:00', 20.215319266294166);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:00:00', 20.81765612016418);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:10:00', 20.771935624088517);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:20:00', 20.944209556384703);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:30:00', 20.610037946505567);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:40:00', 20.81389531856648);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:50:00', 20.195946366739108);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:00:00', 25.66624128668068);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:10:00', 25.391982132308694);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:20:00', 25.84787871226652);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:30:00', 25.600605742306875);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:40:00', 25.35636326043316);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:50:00', 25.225735066913);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:00:00', 25.359039779493045);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:10:00', 25.012426558020447);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:20:00', 25.82713770767008);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:30:00', 25.442540638926115);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:40:00', 25.274139131778046);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:50:00', 25.35556174613059);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:00:00', 25.75836325601163);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:10:00', 25.83775033925234);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:20:00', 25.742874700634285);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:30:00', 25.77207446486909);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:40:00', 25.803784200896395);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:50:00', 25.214425678018586);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:00:00', 25.538252382613273);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:10:00', 25.98189905172387);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:20:00', 25.815837254882865);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:30:00', 25.252159772850863);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:40:00', 25.158702508825808);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:50:00', 25.856533824655532);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:00:00', 25.72820949392227);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:10:00', 25.66981889764483);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:20:00', 25.459337912377826);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:30:00', 25.581434405890104);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:40:00', 25.915241183791146);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:50:00', 25.69607035802048);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:00:00', 25.589296120119702);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:10:00', 25.126513844500145);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:20:00', 25.800878550322214);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:30:00', 25.755925805605784);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:40:00', 25.857677764869447);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:50:00', 25.729061788223557);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:00:00', 25.028572810770456);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:10:00', 25.52433676911993);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:20:00', 25.689712420391746);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:30:00', 25.913576980992943);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:40:00', 25.95352067181972);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:50:00', 25.72743288406616);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:00:00', 25.207218028481336);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:10:00', 25.859512492495718);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:20:00', 25.914536506877173);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:30:00', 25.29069846726668);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:40:00', 25.999551138048073);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:50:00', 25.62359351633416);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:00:00', 25.503930477548035);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:10:00', 25.778348269011715);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:20:00', 25.61358296581122);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:30:00', 25.407082954413692);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:40:00', 25.59356132139061);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:50:00', 25.454359469668102);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:00:00', 25.600497611200172);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:10:00', 25.089287346535247);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:20:00', 25.406186385021478);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:30:00', 25.902770292173233);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:40:00', 25.430122083454933);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:50:00', 25.343677578353137);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:00:00', 25.827782785562448);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:10:00', 25.29812304633534);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:20:00', 25.449369799821127);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:30:00', 25.50067665405068);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:40:00', 25.9979864662843);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:50:00', 25.694267901014552);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:00:00', 25.483896119366477);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:10:00', 25.87093319303245);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:20:00', 25.45105002024843);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:30:00', 25.946018147464862);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:40:00', 25.15067665067811);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:50:00', 25.061446129380773);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:00:00', 25.421278070751292);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:10:00', 25.553324122715736);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:20:00', 25.193917339628353);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:30:00', 25.608770265474906);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:40:00', 25.005467597434503);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:50:00', 25.32181534608147);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:00:00', 25.603882488546358);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:10:00', 25.22053316373271);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:20:00', 25.68500870834277);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:30:00', 25.91769766933929);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:40:00', 25.849312452201982);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:50:00', 25.990145537303054);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:00:00', 25.937698567944693);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:10:00', 25.42855321297195);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:20:00', 25.178883393968068);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:30:00', 25.69575988830021);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:40:00', 25.172342442407984);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:50:00', 25.080605518088852);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:00:00', 25.78950316042253);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:10:00', 25.88138405813117);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:20:00', 25.353321832682024);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:30:00', 25.486967713164372);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:40:00', 25.241081032841397);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:50:00', 25.49196275290994);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:00:00', 25.45332850185998);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:10:00', 25.819180142996192);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:20:00', 25.41239031904037);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:30:00', 25.24367749100779);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:40:00', 30.27412775449895);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:50:00', 30.188777700355352);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:00:00', 30.446072701352517);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:10:00', 30.459609433778173);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:20:00', 30.257042607870012);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:30:00', 30.165281274620337);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:40:00', 30.226078491144943);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:50:00', 30.4460845208148);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:00:00', 30.270296036363806);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:10:00', 30.641048708539444);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:20:00', 30.14382111729586);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:30:00', 30.820109011539174);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:40:00', 30.84493226524782);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:50:00', 30.328083422720017);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:00:00', 30.60516210513217);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:10:00', 30.408059017291606);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:20:00', 30.34669916511969);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:30:00', 30.211917603793683);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:40:00', 30.366852988681565);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:50:00', 30.079332173258585);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:00:00', 30.163940918962364);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:10:00', 30.441891046089662);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:20:00', 30.097050080762052);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:30:00', 30.781660819725904);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:40:00', 30.556900724158172);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:50:00', 30.854915416016965);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:00:00', 30.76581319808981);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:10:00', 30.09515965534585);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:20:00', 30.85011299706575);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:30:00', 30.06167508067504);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:40:00', 30.613744288084394);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:50:00', 30.031460356397986);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:00:00', 30.576925005207194);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:10:00', 30.506954075147476);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:20:00', 30.583027116313506);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:30:00', 30.98952347335216);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:40:00', 30.537960928829108);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:50:00', 30.55906448897572);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:00:00', 30.938395927221197);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:10:00', 30.19125358459283);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:20:00', 30.89218489014141);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:30:00', 30.269881828803516);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:40:00', 30.62453863903956);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:50:00', 30.47049204000605);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:00:00', 30.929074483161077);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:10:00', 30.497084271977826);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:20:00', 30.23546928343591);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:30:00', 30.05088868062539);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:40:00', 30.479104535173093);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:50:00', 30.991924456036607);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:00:00', 30.1321253497226);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:10:00', 30.361037789125465);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:20:00', 30.567939373984228);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:30:00', 30.60608225546851);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:40:00', 30.93910000072389);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:50:00', 30.18435764271089);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:00:00', 30.084093443062766);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:10:00', 30.133540232917728);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:20:00', 30.415224360597637);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:30:00', 30.970271904292225);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:40:00', 30.392442033725175);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:50:00', 30.717345037830334);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:00:00', 30.29287254595604);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:10:00', 30.68001525671801);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:20:00', 30.323324484703765);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:30:00', 30.712905929174017);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:40:00', 30.237340925912207);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:50:00', 30.08066848279619);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:00:00', 30.90308092673425);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:10:00', 30.99790892366655);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:20:00', 30.75279348191689);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:30:00', 30.25620656041354);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:40:00', 30.340489058209528);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:50:00', 30.897623537636733);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:00:00', 30.37151989040234);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:10:00', 30.61349292258621);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:20:00', 30.816884181080642);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:30:00', 30.504656086416432);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:40:00', 30.87450944999693);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:50:00', 30.204642524976364);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:00:00', 30.767957826011852);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:10:00', 30.06720747943294);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:20:00', 30.94352337770597);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:30:00', 30.833975363783203);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:40:00', 30.001082718405442);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:50:00', 30.62722738811884);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:00:00', 30.419754881059927);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:10:00', 30.307988116789986);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:20:00', 30.040255900195287);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:30:00', 30.052270697149037);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:40:00', 30.75465363097227);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:50:00', 30.25257093187005);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:00:00', 30.282273080528512);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:10:00', 30.427327140213684);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:20:00', 30.319335182679588);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:30:00', 30.37032099187025);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:40:00', 30.988634373631932);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:50:00', 30.828145873605614);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:00:00', 30.747086577208773);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:10:00', 30.267317734395885);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:20:00', 35.87736653308371);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:30:00', 35.213688135510274);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:40:00', 35.624399603171256);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:50:00', 35.54411296571213);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:00:00', 35.85002241567013);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:10:00', 35.18600923481531);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:20:00', 35.40299410979962);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:30:00', 35.09347255712887);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:40:00', 35.483986843607994);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:50:00', 35.138503824705126);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:00:00', 35.35293215080399);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:10:00', 35.60403409350849);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:20:00', 35.889313409443915);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:30:00', 35.376640675013235);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:40:00', 35.37812938577163);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:50:00', 35.340828488321115);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:00:00', 35.164237384320586);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:10:00', 35.372113230366274);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:20:00', 35.172726086784465);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:30:00', 35.4062303396216);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:40:00', 35.50774999212543);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:50:00', 35.69456814670008);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:00:00', 35.310291115843164);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:10:00', 35.769803853572924);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:20:00', 35.85937128671548);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:30:00', 35.450095945544774);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:40:00', 35.795843158352746);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:50:00', 35.3576696494664);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:00:00', 35.03003206031267);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:10:00', 35.79711747731257);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:20:00', 35.72215106257537);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:30:00', 35.87982201400974);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:40:00', 35.979599502978346);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:50:00', 35.240513882819045);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:00:00', 35.5556645216999);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:10:00', 35.67801636739599);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:20:00', 35.358328331146204);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:30:00', 35.890864610931736);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:40:00', 35.26681211624377);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:50:00', 35.879822712986986);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:00:00', 35.423068994010364);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:10:00', 35.3031232166475);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:20:00', 35.328828251473276);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:30:00', 35.687884043013135);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:40:00', 35.76018466466917);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:50:00', 35.80132183530506);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:00:00', 35.91475743887499);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:10:00', 35.672540184922454);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:20:00', 35.66439431430592);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:30:00', 35.28120978649861);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:40:00', 35.277339377103594);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:50:00', 35.01572217075871);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:00:00', 35.81763565574464);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:10:00', 35.60486763659024);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:20:00', 35.16818650113106);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:30:00', 35.86560072456925);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:40:00', 35.52020214456461);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:50:00', 35.23640805626655);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:00:00', 35.864600142011156);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:10:00', 35.89457994740151);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:20:00', 35.10484680070225);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:30:00', 35.60555515907971);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:40:00', 35.69632849630121);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:50:00', 35.58037240649336);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:00:00', 35.55190868293294);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:10:00', 35.31340455136142);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:20:00', 35.94962996624675);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:30:00', 35.30671500622937);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:40:00', 35.818865694354976);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:50:00', 35.621550979572355);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:00:00', 35.592736904726955);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:10:00', 35.82557283163196);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:20:00', 35.755573670923816);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:30:00', 35.169409433016675);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:40:00', 35.13349100077494);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:50:00', 35.2909332102636);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:00:00', 35.660308984561546);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:10:00', 35.81682683853163);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:20:00', 35.946497826369246);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:30:00', 35.303284063349686);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:40:00', 35.21681100014162);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:50:00', 35.60441856508883);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:00:00', 35.788542557562636);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:10:00', 35.261086449108404);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:20:00', 35.84207395018552);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:30:00', 35.13737421901026);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:40:00', 35.58797977996542);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:50:00', 35.82303019515759);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:00:00', 35.26929518692598);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:10:00', 35.27228631099635);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:20:00', 35.123823725129014);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:30:00', 35.11246238242373);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:40:00', 35.590660113589934);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:50:00', 35.41334554952506);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:00:00', 35.651952172286485);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:10:00', 35.106181340216715);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:20:00', 35.4567457663772);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:30:00', 35.8275466244328);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:40:00', 35.184825707890894);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:50:00', 35.47984459134186);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:00:00', 40.201337084523104);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:10:00', 40.84734870827102);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:20:00', 40.73705606414366);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:30:00', 40.73252660487228);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:40:00', 40.172410245348146);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:50:00', 40.048828795606454);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:00:00', 40.983784941808956);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:10:00', 40.06515187639006);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:20:00', 40.43276045789533);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:30:00', 40.20081847790017);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:40:00', 40.19821213906828);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:50:00', 40.94956311116983);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:00:00', 40.29817086437369);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:10:00', 40.431938065237006);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:20:00', 40.36603231123525);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:30:00', 40.132146902458615);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:40:00', 40.588114927354994);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:50:00', 40.60736948424093);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:00:00', 40.03099858528583);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:10:00', 40.938607980030326);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:20:00', 40.03294207429186);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:30:00', 40.764112050680716);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:40:00', 40.672025874759676);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:50:00', 40.30584769034546);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:00:00', 40.94998335104924);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:10:00', 40.77396252505749);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:20:00', 40.91530244740165);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:30:00', 40.14370527124542);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:40:00', 40.590636278757756);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:50:00', 40.24156378633403);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:00:00', 40.073198252283404);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:10:00', 40.32188296069385);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:20:00', 40.05834932525424);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:30:00', 40.98009611347602);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:40:00', 40.807081030095226);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:50:00', 40.416308596525546);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:00:00', 40.195406604095496);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:10:00', 40.012208317863994);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:20:00', 40.997351313310965);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:30:00', 40.001139848708185);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:40:00', 40.11660570442965);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:50:00', 40.28369726867717);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:00:00', 40.566731939172016);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:10:00', 40.44968007271433);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:20:00', 40.03636727684547);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:30:00', 40.70737509962399);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:40:00', 40.13617661180935);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:50:00', 40.03591179587625);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:00:00', 40.79305684132999);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:10:00', 40.73273677300458);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:20:00', 40.26359489760031);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:30:00', 40.22100136997964);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:40:00', 40.02540702968399);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:50:00', 40.66516481797407);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:00:00', 40.809816864317504);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:10:00', 40.42502142809004);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:20:00', 40.594666767762455);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:30:00', 40.13449322396993);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:40:00', 40.40481232479665);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:50:00', 40.57203754327355);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:00:00', 40.59849946478817);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:10:00', 40.71961441011085);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:20:00', 40.16141923244062);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:30:00', 40.6512274207581);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:40:00', 40.507970240205545);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:50:00', 40.276511151846144);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:00:00', 40.798481342643385);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:10:00', 40.73265336679539);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:20:00', 40.285347037034136);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:30:00', 40.31497623962321);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:40:00', 40.4246202741558);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:50:00', 40.07422963599871);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:00:00', 40.12958348874789);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:10:00', 40.86818067987339);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:20:00', 40.642069489007156);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:30:00', 40.596095805955095);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:40:00', 40.52245777203022);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:50:00', 40.8366015203503);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:00:00', 40.512374857344334);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:10:00', 40.67906786720226);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:20:00', 40.76433461725766);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:30:00', 40.044167537846505);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:40:00', 40.41353814037301);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:50:00', 40.201682436018416);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:00:00', 40.623608235696494);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:10:00', 40.18297876833851);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:20:00', 40.688843141973095);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:30:00', 40.5891966023212);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:40:00', 40.522140381402785);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:50:00', 40.92296777389312);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:00:00', 40.24515676391933);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:10:00', 40.655731796548345);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:20:00', 40.92931397812652);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:30:00', 40.68752072547414);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:40:00', 40.32399036791148);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:50:00', 40.25529336316137);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:00:00', 40.10505267289531);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:10:00', 40.5226174091793);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:20:00', 40.86920004184902);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:30:00', 40.1906360993813);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:40:00', 45.85960967277176);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:50:00', 45.85471011877021);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:00:00', 45.1989877709722);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:10:00', 45.3504578840903);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:20:00', 45.91299356491165);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:30:00', 45.01371428303895);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:40:00', 45.728166750892846);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:50:00', 45.03541560229253);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:00:00', 45.5986239641956);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:10:00', 45.81923918989048);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:20:00', 45.618445773675745);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:30:00', 45.30979336518993);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:40:00', 45.15934246555148);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:50:00', 45.18658038758665);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:00:00', 45.742299566197865);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:10:00', 45.47496558470984);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:20:00', 45.17284107060644);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:30:00', 45.325728528577365);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:40:00', 45.75261959120912);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:50:00', 45.603756566101396);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:00:00', 45.91894555894857);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:10:00', 45.242366534994744);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:20:00', 45.57919077554701);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:30:00', 45.33679909615693);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:40:00', 45.20538737277241);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:50:00', 45.22278819604679);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:00:00', 45.23833000477602);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:10:00', 45.16959916289375);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:20:00', 45.45263959451466);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:30:00', 45.97908659091804);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:40:00', 45.140983413820756);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:50:00', 45.86277400548062);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:00:00', 45.97219831605039);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:10:00', 45.04580984090806);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:20:00', 45.59774031848686);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:30:00', 45.63530423956679);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:40:00', 45.417438997189045);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:50:00', 45.028671692966654);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:00:00', 45.40639490081234);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:10:00', 45.16304868205777);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:20:00', 45.049404899156826);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:30:00', 45.23123606201219);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:40:00', 45.487575568542034);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:50:00', 45.812505373419285);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:00:00', 45.1998713106634);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:10:00', 45.91470362508454);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:20:00', 45.88477858938986);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:30:00', 45.70057539782315);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:40:00', 45.92421826179503);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:50:00', 45.74893122420723);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:00:00', 45.7943930999712);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:10:00', 45.82325298584645);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:20:00', 45.66569344248397);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:30:00', 45.67425658687078);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:40:00', 45.22746463473383);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:50:00', 45.17154717754609);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:00:00', 45.10854903438189);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:10:00', 45.43473479850994);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:20:00', 45.86222726916496);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:30:00', 45.05271695541182);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:40:00', 45.21940679245271);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:50:00', 45.63379343662304);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:00:00', 45.83825662839634);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:10:00', 45.5920962016897);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:20:00', 45.29665460968132);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:30:00', 45.55980435003492);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:40:00', 45.29945324344045);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:50:00', 45.64509704799861);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:00:00', 45.93516281288697);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:10:00', 45.37087470863014);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:20:00', 45.38291167583321);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:30:00', 45.026662485818065);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:40:00', 45.948698660100106);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:50:00', 45.89348000261725);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:00:00', 45.27649342522346);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:10:00', 45.85725180589345);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:20:00', 45.02114997882529);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:30:00', 45.06405539268047);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:40:00', 45.80949949880207);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:50:00', 45.495786881922356);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:00:00', 45.720225241407);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:10:00', 45.933869209111215);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:20:00', 45.48615887788706);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:30:00', 45.173105831338226);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:40:00', 45.32040103611929);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:50:00', 45.217329152085284);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:00:00', 45.70123831210879);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:10:00', 45.86375901353405);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:20:00', 45.51706924224786);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:30:00', 45.64083981618546);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:40:00', 45.846289361621196);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:50:00', 45.01135727097061);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:00:00', 45.8523614882762);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:10:00', 45.49417126000201);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:20:00', 45.075978494534034);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:30:00', 45.39386395911032);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:40:00', 45.28440126143178);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:50:00', 45.74746698766563);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:00:00', 45.102120344454384);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:10:00', 45.48826104948018);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:20:00', 50.04130376664194);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:30:00', 50.727901163386775);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:40:00', 50.87199601112955);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:50:00', 50.883147649282215);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:00:00', 50.412389559969775);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:10:00', 50.34857424700034);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:20:00', 50.215813927173606);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:30:00', 50.018098469361924);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:40:00', 50.73425346749341);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:50:00', 50.002531690603256);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:00:00', 50.11721308628489);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:10:00', 50.009792988413075);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:20:00', 50.32497669421603);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:30:00', 50.37683573640825);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:40:00', 50.051474942952034);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:50:00', 50.07289474404618);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:00:00', 50.391262596452464);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:10:00', 50.615218842210474);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:20:00', 50.57679177530203);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:30:00', 50.169333861882826);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:40:00', 50.467064754194354);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:50:00', 50.970678718984395);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:00:00', 50.02344103655669);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:10:00', 50.7407045279223);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:20:00', 50.629181974188796);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:30:00', 50.31194882276552);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:40:00', 50.81796755115351);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:50:00', 50.6852589724782);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:00:00', 50.10591157286973);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:10:00', 50.74949084724007);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:20:00', 50.86766143128229);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:30:00', 50.52841254042059);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:40:00', 50.21186794420065);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:50:00', 50.23745113566411);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:00:00', 50.80771016250989);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:10:00', 50.48986958238247);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:20:00', 50.28259222979104);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:30:00', 50.161565650554884);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:40:00', 50.51860921354403);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:50:00', 50.96407846353788);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:00:00', 50.323549622760275);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:10:00', 50.916330159524314);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:20:00', 50.85587878745068);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:30:00', 50.65454538449593);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:40:00', 50.675497805003396);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:50:00', 50.408827223672006);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:00:00', 50.89433758139433);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:10:00', 50.03073008577499);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:20:00', 50.32301210284869);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:30:00', 50.116087557817785);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:40:00', 50.45002574484946);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:50:00', 50.48432457719473);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:00:00', 50.59331195216032);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:10:00', 50.25303251796372);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:20:00', 50.3226011973296);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:30:00', 50.8550816464231);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:40:00', 50.99693944847967);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:50:00', 50.53003507084809);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:00:00', 50.44995606064558);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:10:00', 50.962455062018996);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:20:00', 50.04440554788321);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:30:00', 50.006627699043655);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:40:00', 50.5233631580475);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:50:00', 50.14425594578732);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:00:00', 50.62328506291849);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:10:00', 50.20417859692058);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:20:00', 50.33791801275508);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:30:00', 50.941081823467684);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:40:00', 50.27692920354136);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:50:00', 50.02908356520346);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:00:00', 50.843480719728326);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:10:00', 50.99652467140378);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:20:00', 50.83933192071146);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:30:00', 50.719743522754925);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:40:00', 50.8185853508712);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:50:00', 50.728112920211274);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:00:00', 50.93564338864228);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:10:00', 50.93162421598968);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:20:00', 50.682675687135884);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:30:00', 50.3653949442103);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:40:00', 50.693278186874196);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:50:00', 50.19673957513225);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:00:00', 50.84078682444565);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:10:00', 50.49072691853029);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:20:00', 50.71455809050517);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:30:00', 50.40283576568372);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:40:00', 50.09553898931378);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:50:00', 50.5771581130709);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:00:00', 50.8512641265803);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:10:00', 50.99025618868679);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:20:00', 50.54990302126502);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:30:00', 50.02189647716469);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:40:00', 50.5860172158262);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:50:00', 50.77702741926138);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:00:00', 50.69985640465449);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:10:00', 50.44780132561315);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:20:00', 50.44646297667806);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:30:00', 50.790495880398595);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:40:00', 50.66350390527233);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:50:00', 50.67518770201878);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:00:00', 55.86461671925514);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:10:00', 55.78305592759162);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:20:00', 55.11197041167928);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:30:00', 55.515116152767966);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:40:00', 55.46584376774793);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:50:00', 55.913224447583026);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:00:00', 55.88845127660456);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:10:00', 55.52716976049961);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:20:00', 55.76876652590235);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:30:00', 55.025615294620316);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:40:00', 55.577111109691664);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:50:00', 55.75536154797752);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:00:00', 55.713118360100154);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:10:00', 55.028975567513676);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:20:00', 55.92508590202515);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:30:00', 55.41531042821187);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:40:00', 55.22101104884555);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:50:00', 55.88218816525603);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:00:00', 55.71673883942393);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:10:00', 55.5404089312301);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:20:00', 55.86064551474242);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:30:00', 55.58431548296966);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:40:00', 55.78043236004886);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:50:00', 55.14608087048665);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:00:00', 55.754244136434096);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:10:00', 55.364317872580244);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:20:00', 55.03948319958774);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:30:00', 55.85517637116233);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:40:00', 55.3587811811092);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:50:00', 55.826617148199254);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:00:00', 55.271131376795395);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:10:00', 55.23127750798324);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:20:00', 55.68116746960627);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:30:00', 55.03574324586153);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:40:00', 55.413332995916996);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:50:00', 55.23178134439042);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:00:00', 55.96892214895804);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:10:00', 55.11684222383856);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:20:00', 55.83445791300146);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:30:00', 55.57400857835226);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:40:00', 55.93485961438986);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:50:00', 55.429841228530314);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:00:00', 55.08035236743577);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:10:00', 55.41989113765008);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:20:00', 55.805544979426);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:30:00', 55.95946278699081);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:40:00', 55.54599750274753);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:50:00', 55.53445712721546);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:00:00', 55.16151681479046);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:10:00', 55.8920383082977);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:20:00', 55.72802993943361);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:30:00', 55.25451039355101);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:40:00', 55.99165299725528);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:50:00', 55.141520822030465);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:00:00', 55.817497050028884);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:10:00', 55.5550355233503);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:20:00', 55.05093545638608);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:30:00', 55.298948039581134);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:40:00', 55.96034981420127);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:50:00', 55.25624307611871);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:00:00', 55.18341244297447);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:10:00', 55.65755779828505);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:20:00', 55.62840711299797);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:30:00', 55.44172257257327);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:40:00', 55.94064309141799);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:50:00', 55.0708638460396);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:00:00', 55.014370631496924);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:10:00', 55.561869714864166);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:20:00', 55.997882720027505);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:30:00', 55.53117501273829);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:40:00', 55.12045803801976);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:50:00', 55.5305140970909);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:00:00', 55.18576884352693);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:10:00', 55.368122884800975);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:20:00', 55.46407661104355);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:30:00', 55.6038584617503);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:40:00', 55.10853132281619);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:50:00', 55.13139267312611);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:00:00', 55.69495905228585);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:10:00', 55.42283217515164);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:20:00', 55.60362109918761);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:30:00', 55.94376708230326);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:40:00', 55.7073472033042);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:50:00', 55.34929907927907);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:00:00', 55.14283314247304);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:10:00', 55.736943441535175);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:20:00', 55.174698766698505);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:30:00', 55.487672178112994);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:40:00', 55.10799408252901);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:50:00', 55.315593534425204);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:00:00', 55.59579859772211);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:10:00', 55.45114958941346);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:20:00', 55.66004661399281);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:30:00', 55.71685388314182);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:40:00', 55.068012354321745);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:50:00', 55.654963998005535);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:00:00', 55.17723467236714);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:10:00', 55.75479506184008);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:20:00', 55.15778134521637);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:30:00', 55.795424639230454);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:40:00', 60.42824126662819);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:50:00', 60.5757671418157);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:00:00', 60.1404810626188);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:10:00', 60.509704911130534);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:20:00', 60.646924356417465);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:30:00', 60.006959892353706);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:40:00', 60.77657049248974);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:50:00', 60.46017123295519);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:00:00', 60.48401439712486);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:10:00', 60.66858749935577);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:20:00', 60.96510334799966);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:30:00', 60.1187661935626);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:40:00', 60.89901268733274);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:50:00', 60.7100584149522);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:00:00', 60.343916158147145);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:10:00', 60.777117979126956);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:20:00', 60.73457150634369);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:30:00', 60.5304253179845);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:40:00', 60.72169834087641);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:50:00', 60.965641344091345);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:00:00', 60.217044491210146);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:10:00', 60.06542975986038);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:20:00', 60.32427943162242);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:30:00', 60.906465781861016);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:40:00', 60.461791464153684);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:50:00', 60.90000624945958);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:00:00', 60.24177596799514);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:10:00', 60.14024047215926);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:20:00', 60.98749731612882);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:30:00', 60.08592227499234);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:40:00', 60.35343869161658);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:50:00', 60.13885248255164);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:00:00', 60.35450042684454);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:10:00', 60.03211618097429);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:20:00', 60.273187737561074);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:30:00', 60.305472400797136);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:40:00', 60.59858406547435);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:50:00', 60.716684350836275);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:00:00', 60.40073723472085);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:10:00', 60.332361240318995);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:20:00', 60.19975296613247);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:30:00', 60.94185056034696);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:40:00', 60.27306202553091);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:50:00', 60.419961754944936);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:00:00', 60.68711857415286);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:10:00', 60.308048563696616);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:20:00', 60.43288153503136);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:30:00', 60.22240610394697);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:40:00', 60.87432112575496);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:50:00', 60.42657024373679);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:00:00', 60.437605292397);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:10:00', 60.09059057790051);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:20:00', 60.280926312745045);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:30:00', 60.228427584432715);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:40:00', 60.95304592243521);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:50:00', 60.54682796239075);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:00:00', 60.46454523913678);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:10:00', 60.51809831891373);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:20:00', 60.076953460188854);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:30:00', 60.499199498516006);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:40:00', 60.452895701794034);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:50:00', 60.20413704551565);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:00:00', 60.07591539489422);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:10:00', 60.63127640945167);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:20:00', 60.38481450413325);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:30:00', 60.70560320729101);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:40:00', 60.91809569688567);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:50:00', 60.383938836598844);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:00:00', 60.67010478271658);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:10:00', 60.42291672933783);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:20:00', 60.31070562512965);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:30:00', 60.25651888575585);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:40:00', 60.36355352853065);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:50:00', 60.623416188823);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:00:00', 60.29259214493167);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:10:00', 60.49130446276643);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:20:00', 60.505422467667636);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:30:00', 60.582644388830175);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:40:00', 60.71386345313682);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:50:00', 60.931482176620534);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:00:00', 60.38112402586485);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:10:00', 60.99702588462689);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:20:00', 60.03308034897499);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:30:00', 60.05761908007053);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:40:00', 60.1214402114995);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:50:00', 60.88269160882629);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:00:00', 60.80626185025086);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:10:00', 60.774934578921076);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:20:00', 60.823458712724495);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:30:00', 60.52168862595384);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:40:00', 60.73565510163873);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:50:00', 60.60350497776844);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:00:00', 60.68478473293711);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:10:00', 60.346168541196306);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:20:00', 60.9149565352909);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:30:00', 60.28945564131935);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:40:00', 60.26923362436257);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:50:00', 60.01858550817403);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:00:00', 60.17470875147571);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:10:00', 60.604185712804565);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:20:00', 65.75717243160803);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:30:00', 65.07422746865474);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:40:00', 65.46395803181234);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:50:00', 65.32164179463363);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:00:00', 65.97344273308903);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:10:00', 65.16434989087553);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:20:00', 65.37884619243778);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:30:00', 65.97414561220593);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:40:00', 65.573338910014);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:50:00', 65.32982095211847);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:00:00', 65.11231290469598);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:10:00', 65.92361485867106);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:20:00', 65.37560800677892);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:30:00', 65.71195976194832);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:40:00', 65.262703659211);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:50:00', 65.15018180869143);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:00:00', 65.65495183113529);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:10:00', 65.05729917148591);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:20:00', 65.36401053057132);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:30:00', 65.63085936740404);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:40:00', 65.98756212918181);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:50:00', 65.75438329668366);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:00:00', 65.73123308297751);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:10:00', 65.3516908718983);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:20:00', 65.53636095470429);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:30:00', 65.08651350564215);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:40:00', 65.21751895177366);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:50:00', 65.3719973788212);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:00:00', 65.6405198875497);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:10:00', 65.13481851433889);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:20:00', 65.90554867795352);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:30:00', 65.08230521755553);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:40:00', 65.23338715435199);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:50:00', 65.81721113896879);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:00:00', 65.92490658172284);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:10:00', 65.8295466301566);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:20:00', 65.2709442164126);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:30:00', 65.67910457842821);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:40:00', 65.60570866531853);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:50:00', 65.18723696283372);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:00:00', 65.31752146342961);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:10:00', 65.69913903281);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:20:00', 65.02145958030293);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:30:00', 65.61372709361585);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:40:00', 65.44635354489421);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:50:00', 65.62965622782808);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:00:00', 65.35589597584978);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:10:00', 65.41784713643604);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:20:00', 65.18958358892179);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:30:00', 65.36613160951111);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:40:00', 65.50508385443378);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:50:00', 65.1803100219306);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:00:00', 65.05908329669485);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:10:00', 65.18186863371146);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:20:00', 65.61694445144289);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:30:00', 65.79260171668231);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:40:00', 65.2952654373761);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:50:00', 65.45914597305739);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:00:00', 65.01153576684446);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:10:00', 65.74243000191585);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:20:00', 65.81355219156411);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:30:00', 65.4047858511711);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:40:00', 65.27522667732593);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:50:00', 65.34709539941646);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:00:00', 65.8323582407959);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:10:00', 65.28052350397084);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:20:00', 65.21648824569458);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:30:00', 65.74454725415396);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:40:00', 65.94040871614496);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:50:00', 65.35980720632816);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:00:00', 65.01297183319355);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:10:00', 65.03731826900781);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:20:00', 65.96657288125054);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:30:00', 65.11871187478256);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:40:00', 65.60985795474032);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:50:00', 65.48919459176929);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:00:00', 65.908713534529);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:10:00', 65.32591704286483);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:20:00', 65.93247438584325);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:30:00', 65.25551421491285);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:40:00', 65.73367349897329);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:50:00', 65.07091664126513);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:00:00', 65.07512162349447);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:10:00', 65.7151484417674);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:20:00', 65.43895169098508);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:30:00', 65.1805144957315);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:40:00', 65.70261769381938);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:50:00', 65.56150506790509);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:00:00', 65.23764142715255);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:10:00', 65.96850090859809);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:20:00', 65.60991514907558);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:30:00', 65.0137033690365);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:40:00', 65.55642960976178);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:50:00', 65.4291480698173);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:00:00', 65.7948850825054);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:10:00', 65.68857036904905);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:20:00', 65.88982844642518);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:30:00', 65.6797902956207);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:40:00', 65.04828322798917);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:50:00', 65.98121258744061);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:00:00', 70.84792375176262);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:10:00', 70.24714401393534);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:20:00', 70.82080353559094);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:30:00', 70.22493475440335);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:40:00', 70.83517576657364);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:50:00', 70.48220481199782);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:00:00', 70.50987238063006);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:10:00', 70.65887788836065);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:20:00', 70.15813700129239);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:30:00', 70.93666891618649);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:40:00', 70.09444659260299);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:50:00', 70.44610246379736);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:00:00', 70.60755376156217);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:10:00', 70.62952517971817);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:20:00', 70.8341430461983);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:30:00', 70.54258873073861);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:40:00', 70.78584097729174);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:50:00', 70.56133790996552);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:00:00', 70.00926091252859);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:10:00', 70.84692164609287);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:20:00', 70.81560829042392);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:30:00', 70.4564813058073);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:40:00', 70.53922518502625);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:50:00', 70.29454246182891);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:00:00', 70.62694229701485);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:10:00', 70.77008506994437);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:20:00', 70.90039896257827);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:30:00', 70.20802685112797);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:40:00', 70.23043617175695);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:50:00', 70.93868928146462);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:00:00', 70.32214900023658);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:10:00', 70.98721417278793);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:20:00', 70.7073862675702);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:30:00', 70.07073125003258);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:40:00', 70.21898268089228);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:50:00', 70.0459327620224);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:00:00', 70.47883044279872);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:10:00', 70.0357550187124);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:20:00', 70.9029467289573);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:30:00', 70.19262260368063);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:40:00', 70.67181564949811);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:50:00', 70.54165230053579);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:00:00', 70.20576791812435);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:10:00', 70.97042118947293);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:20:00', 70.55995951405272);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:30:00', 70.59708865561609);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:40:00', 70.61280063207539);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:50:00', 70.11275312715213);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:00:00', 70.40198693106453);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:10:00', 70.52719613265567);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:20:00', 70.36503580697642);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:30:00', 70.35992503082046);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:40:00', 70.89632985223194);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:50:00', 70.31524736062875);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:00:00', 70.14663551460536);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:10:00', 70.629373723081);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:20:00', 70.45578605413314);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:30:00', 70.37049605560195);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:40:00', 70.33728568741186);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:50:00', 70.0374832297144);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:00:00', 70.46953234119924);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:10:00', 70.44757344459161);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:20:00', 70.61022791965419);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:30:00', 70.44489295012828);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:40:00', 70.91419584310704);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:50:00', 70.14092066876647);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:00:00', 70.49788868528776);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:10:00', 70.2505423822712);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:20:00', 70.82541110065833);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:30:00', 70.49822564109441);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:40:00', 70.23604429880052);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:50:00', 70.47028795047646);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:00:00', 70.52196524379626);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:10:00', 70.73173576741272);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:20:00', 70.59085505060818);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:30:00', 70.61416398517015);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:40:00', 70.35856993530311);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:50:00', 70.79628009722434);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:00:00', 70.39124910067879);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:10:00', 70.3169644539236);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:20:00', 70.66814559254944);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:30:00', 70.66727306779204);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:40:00', 70.59748311984364);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:50:00', 70.63137510919809);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:00:00', 70.15372199828549);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:10:00', 70.94534209063247);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:20:00', 70.11126223644388);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:30:00', 70.62673987071862);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:40:00', 70.55893655546298);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:50:00', 70.85913955652822);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:00:00', 70.44300398324273);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:10:00', 70.92669544436872);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:20:00', 70.68500041259618);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:30:00', 70.73821146363592);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:40:00', 70.09185238051528);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:50:00', 70.52442445866551);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:00:00', 70.76453775181325);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:10:00', 70.87844349736861);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:20:00', 70.10201299228585);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:30:00', 70.73573441334291);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:40:00', 75.73875085565189);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:50:00', 75.68924076910906);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:00:00', 75.40249603606367);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:10:00', 75.10192311249615);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:20:00', 75.23763558581453);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:30:00', 75.33174535792007);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:40:00', 75.13750709632737);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:50:00', 75.87624497874643);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:00:00', 75.51507412723397);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:10:00', 75.17984288750057);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:20:00', 75.57942666194583);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:30:00', 75.17559970453412);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:40:00', 75.57867691398481);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:50:00', 75.5768138086326);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:00:00', 75.18609446940691);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:10:00', 75.23104725966235);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:20:00', 75.0406505098033);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:30:00', 75.10486663190895);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:40:00', 75.91293660240508);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:50:00', 75.48879327914106);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:00:00', 75.87941307619892);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:10:00', 75.59084950346809);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:20:00', 75.1040794971801);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:30:00', 75.30164181144724);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:40:00', 75.77590817467764);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:50:00', 75.6942164667002);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:00:00', 75.94869729038744);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:10:00', 75.72456322147062);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:20:00', 75.04615245346291);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:30:00', 75.08142037966856);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:40:00', 75.82421514583359);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:50:00', 75.99785306189705);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:00:00', 75.68095826431698);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:10:00', 75.81478375577895);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:20:00', 75.43084888893065);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:30:00', 75.3865179272271);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:40:00', 75.85559764019484);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:50:00', 75.3179213371592);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:00:00', 75.99517990722951);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:10:00', 75.77330732423295);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:20:00', 75.41675859559655);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:30:00', 75.61660863738516);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:40:00', 75.29748058931841);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:50:00', 75.86044161935821);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:00:00', 75.68189971766895);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:10:00', 75.52662363332475);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:20:00', 75.99381551449784);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:30:00', 75.09372986941132);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:40:00', 75.76333867961407);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:50:00', 75.98746157570056);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:00:00', 75.59844485443448);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:10:00', 75.11850856645286);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:20:00', 75.85650339910504);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:30:00', 75.73934452601394);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:40:00', 75.70413465820144);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:50:00', 75.73277653616444);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:00:00', 75.52897698618867);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:10:00', 75.06683383389577);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:20:00', 75.10826702568332);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:30:00', 75.92204167961121);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:40:00', 75.67334637926871);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:50:00', 75.0098275704047);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:00:00', 75.79034949378887);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:10:00', 75.25034943540432);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:20:00', 75.2418857167868);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:30:00', 75.55151921991957);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:40:00', 75.22659343226981);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:50:00', 75.6297760503289);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:00:00', 75.60625964704498);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:10:00', 75.92514383190209);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:20:00', 75.33584156667379);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:30:00', 75.45888705086357);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:40:00', 75.1171019402213);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:50:00', 75.34421798283691);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:00:00', 75.70906630644397);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:10:00', 75.13314786525794);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:20:00', 75.82994660760363);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:30:00', 75.05586028954062);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:40:00', 75.72739447197894);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:50:00', 75.27139953673046);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:00:00', 75.0373887193959);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:10:00', 75.58664575769784);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:20:00', 75.64542070383824);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:30:00', 75.93361623534472);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:40:00', 75.21727104704924);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:50:00', 75.39678815950646);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:00:00', 75.09570509331364);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:10:00', 75.97729622254738);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:20:00', 75.0855718432743);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:30:00', 75.97580472209289);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:40:00', 75.42239611691534);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:50:00', 75.65532024535044);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:00:00', 75.78582977910041);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:10:00', 75.49909674660347);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:20:00', 75.61170402506708);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:30:00', 75.30708138689313);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:40:00', 75.45874058634112);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:50:00', 75.67144979152366);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:00:00', 75.4767990754182);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:10:00', 75.03148874161155);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:20:00', 80.05177827557155);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:30:00', 80.55125618457271);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:40:00', 80.56327881372859);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:50:00', 80.62561917424894);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:00:00', 80.13358094872449);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:10:00', 80.57447479351923);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:20:00', 80.38762041089727);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:30:00', 80.22345147063619);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:40:00', 80.77662293919255);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:50:00', 80.06086260124913);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:00:00', 80.20648608787332);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:10:00', 80.24904308099917);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:20:00', 80.4470358500717);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:30:00', 80.10208054257717);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:40:00', 80.6948625345203);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:50:00', 80.20134542326788);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:00:00', 80.63746559248813);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:10:00', 80.09679600352392);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:20:00', 80.40261514796043);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:30:00', 80.35729868236676);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:40:00', 80.68933335226585);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:50:00', 80.62613946216034);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:00:00', 80.13081628909515);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:10:00', 80.83351790238845);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:20:00', 80.15002651009729);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:30:00', 80.6848971808227);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:40:00', 80.17155394414425);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:50:00', 80.01414372310468);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:00:00', 80.23153708385941);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:10:00', 80.1491872075093);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:20:00', 80.86763964224257);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:30:00', 80.27968860134462);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:40:00', 80.46846001239791);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:50:00', 80.7377358061797);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:00:00', 80.06297519616069);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:10:00', 80.47599450773214);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:20:00', 80.16928442772286);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:30:00', 80.1866406554142);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:40:00', 80.26220108849837);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:50:00', 80.80104515673855);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:00:00', 80.02502473593107);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:10:00', 80.65587704369372);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:20:00', 80.43734579422477);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:30:00', 80.4605157182123);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:40:00', 80.07320618895673);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:50:00', 80.83280728273998);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:00:00', 80.52997403844681);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:10:00', 80.47436185006217);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:20:00', 80.61573331558482);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:30:00', 80.53430778954069);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:40:00', 80.46207010848349);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:50:00', 80.10951598700812);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:00:00', 80.14204891643658);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:10:00', 80.37656701517018);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:20:00', 80.68481469910469);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:30:00', 80.78105279409999);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:40:00', 80.9563652599067);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:50:00', 80.72225111864762);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:00:00', 80.53065993656942);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:10:00', 80.93882701354313);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:20:00', 80.19051555622741);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:30:00', 80.05288737619873);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:40:00', 80.20363409046747);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:50:00', 80.00490482513175);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:00:00', 80.1987350455192);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:10:00', 80.72845003892684);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:20:00', 80.40568741749185);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:30:00', 80.8944897444389);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:40:00', 80.13821234591664);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:50:00', 80.80735853124939);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:00:00', 80.26587482706799);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:10:00', 80.28707259941027);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:20:00', 80.29978568164628);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:30:00', 80.19257750117515);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:40:00', 80.51644031350376);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:50:00', 80.20870394717966);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:00:00', 80.04208772645019);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:10:00', 80.74839009450095);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:20:00', 80.74270770219081);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:30:00', 80.27061805307784);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:40:00', 80.2121638737193);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:50:00', 80.44559311165203);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:00:00', 80.96841302878805);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:10:00', 80.62565250431959);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:20:00', 80.83997237788287);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:30:00', 80.83476992395882);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:40:00', 80.33982309292188);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:50:00', 80.6145748758383);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:00:00', 80.74159692216102);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:10:00', 80.23137179819544);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:20:00', 80.68413946439128);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:30:00', 80.60253970493986);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:40:00', 80.38803395921579);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:50:00', 80.41587051820059);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:00:00', 80.63414747366352);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:10:00', 80.17821559821772);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:20:00', 80.18485850974864);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:30:00', 80.96882915978566);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:40:00', 80.25494618536747);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:50:00', 80.47145006805054);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:00:00', 85.07014084281235);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:10:00', 85.36053530006474);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:20:00', 85.72638672081307);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:30:00', 85.68543179856624);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:40:00', 85.06573949110016);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:50:00', 85.39619525156424);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:00:00', 85.34323892038135);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:10:00', 85.83612931876877);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:20:00', 85.38899679126838);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:30:00', 85.5845404775761);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:40:00', 85.49266890254829);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:50:00', 85.2020961319447);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:00:00', 85.70732433516498);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:10:00', 85.45316638950072);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:20:00', 85.60159799035198);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:30:00', 85.48513567233881);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:40:00', 85.72015763104838);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:50:00', 85.27140961968638);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:00:00', 85.06114371615334);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:10:00', 85.68284915594599);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:20:00', 85.86224158293518);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:30:00', 85.03062371816587);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:40:00', 85.73342399005355);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:50:00', 85.29459639238314);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:00:00', 85.4034171818368);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:10:00', 85.99601629298941);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:20:00', 85.58811032293947);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:30:00', 85.52973763102459);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:40:00', 85.56355686500426);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:50:00', 85.89108922003679);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:00:00', 85.9833866146756);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:10:00', 85.06044404268381);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:20:00', 85.77748280834021);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:30:00', 85.6837610301493);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:40:00', 85.15219723566939);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:50:00', 85.10435743280738);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:00:00', 85.37547815372379);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:10:00', 85.77748123606732);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:20:00', 85.54794351256511);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:30:00', 85.45151980009891);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:40:00', 85.39295800693596);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:50:00', 85.06802277138283);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:00:00', 85.55971158029409);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:10:00', 85.38164289249396);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:20:00', 85.10085197592385);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:30:00', 85.48725534100402);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:40:00', 85.52593256171464);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:50:00', 85.87156513298041);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:00:00', 85.07862802922249);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:10:00', 85.72336526868045);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:20:00', 85.82142217076826);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:30:00', 85.57039253696205);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:40:00', 85.12543146754433);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:50:00', 85.95003848846102);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:00:00', 85.93983330592356);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:10:00', 85.8894598818847);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:20:00', 85.55443573823122);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:30:00', 85.75699779977316);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:40:00', 85.66753770050688);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:50:00', 85.30928352007138);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:00:00', 85.66903513434067);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:10:00', 85.083443919299);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:20:00', 85.79800247593478);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:30:00', 85.01679441203906);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:40:00', 85.40328635972477);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:50:00', 85.36241590940696);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:00:00', 85.17806212490461);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:10:00', 85.78086871874406);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:20:00', 85.86432022718121);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:30:00', 85.14542697715467);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:40:00', 85.78580299913301);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:50:00', 85.79107965449309);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:00:00', 85.85390702180145);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:10:00', 85.83790102951062);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:20:00', 85.05869397189487);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:30:00', 85.42097335726912);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:40:00', 85.90493345730768);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:50:00', 85.57236725902774);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:00:00', 85.76671318462299);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:10:00', 85.9854775354553);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:20:00', 85.11491748659687);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:30:00', 85.65618507697398);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:40:00', 85.2976768974066);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:50:00', 85.790727502414);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:00:00', 85.31097224584342);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:10:00', 85.45443176058593);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:20:00', 85.4497890737567);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:30:00', 85.76434428610645);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:40:00', 85.95723995051942);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:50:00', 85.80410985528856);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:00:00', 85.42796805505756);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:10:00', 85.57439837703916);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:20:00', 85.55171629202331);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:30:00', 85.94574567876707);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:40:00', 85.84253228342556);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:50:00', 85.8231956362723);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:00:00', 85.56463113744121);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:10:00', 85.37280340640424);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:20:00', 85.08333657035288);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:30:00', 85.79176703962894);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:40:00', 90.9824625344212);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:50:00', 90.80670841671619);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:00:00', 90.95126333998554);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:10:00', 90.48883720548729);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:20:00', 90.2923260087112);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:30:00', 90.51452461979704);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:40:00', 90.69410598821828);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:50:00', 90.53662585457249);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:00:00', 90.28384547135903);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:10:00', 90.72836694448385);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:20:00', 90.79370407752636);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:30:00', 90.34427946065428);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:40:00', 90.91749139074152);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:50:00', 90.78235016294744);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:00:00', 90.81669047135937);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:10:00', 90.99144445409335);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:20:00', 90.05074397237844);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:30:00', 90.23587629854755);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:40:00', 90.42735397359165);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:50:00', 90.63333890728542);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:00:00', 90.45749785824628);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:10:00', 90.47511827899672);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:20:00', 90.90202603264449);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:30:00', 90.91886052209719);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:40:00', 90.82261955614763);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:50:00', 90.60251735636996);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:00:00', 90.60858268969284);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:10:00', 90.6901637526175);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:20:00', 90.79931191877024);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:30:00', 90.94564613566642);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:40:00', 90.52138817215868);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:50:00', 90.28452056335456);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:00:00', 90.0320292854833);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:10:00', 90.17065842730513);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:20:00', 90.56179378053595);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:30:00', 90.80209675698634);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:40:00', 90.51980296587465);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:50:00', 90.81073939488728);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:00:00', 90.21851907374479);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:10:00', 90.3350149456988);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:20:00', 90.93990581260924);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:30:00', 90.70897406397431);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:40:00', 90.00175599098391);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:50:00', 90.25201589123915);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:00:00', 90.20684508994745);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:10:00', 90.30766125369664);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:20:00', 90.74912066619444);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:30:00', 90.46841740749424);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:40:00', 90.59370522649891);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:50:00', 90.2501217650573);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:00:00', 90.55961793814348);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:10:00', 90.27369721629333);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:20:00', 90.15913834803024);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:30:00', 90.24824580229117);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:40:00', 90.01527042304376);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:50:00', 90.78483738088785);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:00:00', 90.20246578626534);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:10:00', 90.42122852088525);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:20:00', 90.96079390424096);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:30:00', 90.39849247285359);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:40:00', 90.57641891985936);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:50:00', 90.11550147649305);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:00:00', 90.58461185786881);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:10:00', 90.02777344004706);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:20:00', 90.55556257005077);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:30:00', 90.83971627172713);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:40:00', 90.18166451119238);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:50:00', 90.625356986698);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:00:00', 90.59600460482297);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:10:00', 90.00920036129621);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:20:00', 90.26312938191263);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:30:00', 90.21126188660082);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:40:00', 90.27358248872356);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:50:00', 90.65075765804363);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:00:00', 90.56581960625195);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:10:00', 90.30095213443592);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:20:00', 90.5733945270204);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:30:00', 90.22087591168156);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:40:00', 90.70638255635788);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:50:00', 90.51904534190113);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:00:00', 90.93288107407612);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:10:00', 90.02738882847706);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:20:00', 90.94145397447491);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:30:00', 90.49313500741732);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:40:00', 90.99100815129243);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:50:00', 90.93467490108415);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:00:00', 90.1349313909047);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:10:00', 90.29207422425257);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:20:00', 90.71015752581968);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:30:00', 90.2276461972035);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:40:00', 90.24617387551187);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:50:00', 90.39927626803959);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:00:00', 90.30233537599464);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:10:00', 90.17441461736765);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:20:00', 90.78714162395201);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:30:00', 90.44790297372845);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:40:00', 90.7742848703115);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:50:00', 90.9967671844167);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:00:00', 90.29369758387823);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:10:00', 90.85977760577518);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:20:00', 95.70273919522026);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:30:00', 95.72655193466684);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:40:00', 95.68564644336149);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:50:00', 95.23140811939676);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:00:00', 95.41132878464948);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:10:00', 95.51410522097108);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:20:00', 95.12185543853779);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:30:00', 95.21711983477532);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:40:00', 95.08595493204972);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:50:00', 95.2690559077771);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:00:00', 95.8247084729692);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:10:00', 95.37897575959448);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:20:00', 95.9912168689898);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:30:00', 95.64427550907007);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:40:00', 95.53399676703508);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:50:00', 95.5746199904719);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:00:00', 95.9187378270099);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:10:00', 95.39134339392945);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:20:00', 95.7053585881445);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:30:00', 95.34373841660249);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:40:00', 95.78505865015656);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:50:00', 95.35279032777069);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:00:00', 95.36601508459272);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:10:00', 95.16843373559405);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:20:00', 95.14675336545551);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:30:00', 95.59360876887017);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:40:00', 95.05465673216129);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:50:00', 95.08114900094135);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:00:00', 95.20328901548919);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:10:00', 95.11776422317558);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:20:00', 95.09685773302864);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:30:00', 95.44575525609163);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:40:00', 95.76418151133755);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:50:00', 95.9433408206822);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:00:00', 95.43089972426745);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:10:00', 95.10083246997101);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:20:00', 95.31373144887291);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:30:00', 95.49404831529185);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:40:00', 95.12094902570072);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:50:00', 95.14421426311722);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:00:00', 95.03300409670959);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:10:00', 95.5977018635513);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:20:00', 95.20245201405494);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:30:00', 95.8042814980708);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:40:00', 95.80967702428563);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:50:00', 95.6310032550752);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:00:00', 95.78813452304236);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:10:00', 95.3710756559026);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:20:00', 95.84073139666731);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:30:00', 95.87565958347292);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:40:00', 95.27923779126085);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:50:00', 95.43791310863152);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:00:00', 95.37353522312424);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:10:00', 95.45483514345189);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:20:00', 95.27243362298515);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:30:00', 95.53472611364586);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:40:00', 95.9943171036817);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:50:00', 95.75241201361489);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:00:00', 95.662009154259);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:10:00', 95.8559140171521);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:20:00', 95.99341595419622);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:30:00', 95.96512948740204);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:40:00', 95.40885203737166);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:50:00', 95.40368419454084);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:00:00', 95.14626703644213);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:10:00', 95.16021693202427);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:20:00', 95.48791912324057);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:30:00', 95.86748213114836);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:40:00', 95.29693902248422);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:50:00', 95.74430025147372);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:00:00', 95.33353539080912);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:10:00', 95.24171867670813);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:20:00', 95.41393555195542);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:30:00', 95.24211730892031);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:40:00', 95.01001258896444);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:50:00', 95.22691557758905);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:00:00', 95.00702581134178);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:10:00', 95.24259265108412);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:20:00', 95.00263139795263);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:30:00', 95.1527736215333);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:40:00', 95.20922450679696);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:50:00', 95.63795697028615);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:00:00', 95.71452083922787);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:10:00', 95.7978817600075);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:20:00', 95.78574938389043);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:30:00', 95.3894335508658);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:40:00', 95.47191602261879);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:50:00', 95.4161076668443);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:00:00', 95.28418844609357);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:10:00', 95.63241694113232);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:20:00', 95.93718662199717);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:30:00', 95.27693030242013);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:40:00', 95.94671082905306);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:50:00', 95.63096731898416);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:00:00', 95.70082329449929);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:10:00', 95.28902281965944);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:20:00', 95.3096473200545);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:30:00', 95.25709498620873);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:40:00', 95.76420220486904);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:50:00', 95.25611173480905);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:00:00', 100.06185651453708);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:10:00', 100.4580955110936);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:20:00', 100.02486306345648);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:30:00', 100.65935078621763);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:40:00', 100.80202938991845);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:50:00', 100.838535680746);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:00:00', 100.15437846704823);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:10:00', 100.00266102472185);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:20:00', 100.23413928641735);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:30:00', 100.9436158972604);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:40:00', 100.67054972371241);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:50:00', 100.21319010879192);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:00:00', 100.06564362855326);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:10:00', 100.25497290102619);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:20:00', 100.75353786140448);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:30:00', 100.2756056686153);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:40:00', 100.43988404887469);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:50:00', 100.24517273423756);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:00:00', 100.56119548395081);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:10:00', 100.9423620903701);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:20:00', 100.22213644715453);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:30:00', 100.07951390032152);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:40:00', 100.20968090775213);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:50:00', 100.78367715139623);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:00:00', 100.86252769643005);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:10:00', 100.78535813702729);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:20:00', 100.49325730700184);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:30:00', 100.03124276082373);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:40:00', 100.73817148850584);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:50:00', 100.54620495076551);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:00:00', 100.19007432703216);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:10:00', 100.69654359638416);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:20:00', 100.76761046634724);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:30:00', 100.56329351628469);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:40:00', 100.02109622349658);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:50:00', 100.41610251701651);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:00:00', 100.38134443340543);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:10:00', 100.52017412664446);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:20:00', 100.55825864280058);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:30:00', 100.93612685898168);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:40:00', 100.87176024259242);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:50:00', 100.09432075442085);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:00:00', 100.21522312629112);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:10:00', 100.0641885113497);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:20:00', 100.68295526141016);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:30:00', 100.64919442070853);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:40:00', 100.8513673974551);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:50:00', 100.57327938047924);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:00:00', 100.87063567639632);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:10:00', 100.26801292992012);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:20:00', 100.03971650334793);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:30:00', 100.06263305061634);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:40:00', 100.05275073481121);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:50:00', 100.72089880683666);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:00:00', 100.72130967371119);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:10:00', 100.46428694436828);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:20:00', 100.67646105198568);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:30:00', 100.65714458595468);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:40:00', 100.90429981851341);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:50:00', 100.62413165850428);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:00:00', 100.5684317347736);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:10:00', 100.48164986723944);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:20:00', 100.07108879370205);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:30:00', 100.78045834326109);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:40:00', 100.47189403300331);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:50:00', 100.0871521681507);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:00:00', 100.20977695685312);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:10:00', 100.58869246991378);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:20:00', 100.050495427379);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:30:00', 100.07176411567735);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:40:00', 100.66547261466386);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:50:00', 100.6255695225099);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:00:00', 100.52528545450022);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:10:00', 100.06630482528924);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:20:00', 100.54826862079224);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:30:00', 100.27743208948088);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:40:00', 100.59151187969195);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:50:00', 100.19357087141566);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:00:00', 100.2055761518432);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:10:00', 100.52294916116287);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:20:00', 100.22918892587309);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:30:00', 100.30122764892819);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:40:00', 100.46434409024792);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:50:00', 100.09147329074148);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:00:00', 100.673016061652);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:10:00', 100.05183192923042);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:20:00', 100.84874371063628);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:30:00', 100.97796658012604);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:40:00', 100.47187175409692);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:50:00', 100.04795930073355);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:00:00', 100.14495201018543);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:10:00', 100.7712073637323);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:20:00', 100.48376325881216);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:30:00', 100.02989656687107);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:40:00', 100.30850145078257);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:50:00', 100.79732770188521);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:00:00', 100.87191486590562);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:10:00', 100.99561345408628);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:20:00', 100.44482234718608);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:30:00', 100.54045807024269);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:40:00', 105.94962782322027);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:50:00', 105.98419551485628);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:00:00', 105.98450691349268);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:10:00', 105.30243117020771);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:20:00', 105.62442237163904);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:30:00', 105.19655848954201);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:40:00', 105.2678560621535);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:50:00', 105.52085156046782);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:00:00', 105.73739469337072);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:10:00', 105.43309356793623);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:20:00', 105.3890473200834);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:30:00', 105.30614100119176);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:40:00', 105.1108971300237);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:50:00', 105.39575816449894);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:00:00', 105.05690563154423);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:10:00', 105.0661670292193);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:20:00', 105.58161836708838);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:30:00', 105.04807950700233);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:40:00', 105.2419395813828);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:50:00', 105.71938038146294);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:00:00', 105.2768557272797);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:10:00', 105.32026065010504);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:20:00', 105.78523929626823);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:30:00', 105.276924615787);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:40:00', 105.17367190128779);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:50:00', 105.7840852531348);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:00:00', 105.12486918168095);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:10:00', 105.87540477798387);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:20:00', 105.71914313529292);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:30:00', 105.90351017547114);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:40:00', 105.92546541917544);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:50:00', 105.95212937011351);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:00:00', 105.9877572102225);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:10:00', 105.00118159423285);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:20:00', 105.28564303203548);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:30:00', 105.1025275628095);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:40:00', 105.61896840936787);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:50:00', 105.18417419890764);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:00:00', 105.67061022532066);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:10:00', 105.47663142214905);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:20:00', 105.35190138496912);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:30:00', 105.90842581892744);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:40:00', 105.4222231037005);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:50:00', 105.76550949712649);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:00:00', 105.38575269295369);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:10:00', 105.87935680249542);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:20:00', 105.29297243716174);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:30:00', 105.31400347205394);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:40:00', 105.52715002910543);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:50:00', 105.78276795303265);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:00:00', 105.48506256143213);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:10:00', 105.72614615283683);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:20:00', 105.85079254979068);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:30:00', 105.09790941235916);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:40:00', 105.46933283027454);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:50:00', 105.70831466965087);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:00:00', 105.39008927983515);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:10:00', 105.1467233401007);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:20:00', 105.7448854928425);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:30:00', 105.40150169225193);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:40:00', 105.10797364238701);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:50:00', 105.4149393297828);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:00:00', 105.34898977354796);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:10:00', 105.71491489922285);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:20:00', 105.42653168477068);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:30:00', 105.54718526304302);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:40:00', 105.42521382345095);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:50:00', 105.67102658209608);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:00:00', 105.46038317508213);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:10:00', 105.79683626991262);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:20:00', 105.36926571740987);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:30:00', 105.28023685231967);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:40:00', 105.82166563832027);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:50:00', 105.29457431726064);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:00:00', 105.1349211067932);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:10:00', 105.6386492093234);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:20:00', 105.81027712583908);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:30:00', 105.59457413075762);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:40:00', 105.38090232975182);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:50:00', 105.89491581437031);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:00:00', 105.91463360016053);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:10:00', 105.92926587130607);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:20:00', 105.0471923488872);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:30:00', 105.69350925950778);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:40:00', 105.9500247412264);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:50:00', 105.52697621217106);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:00:00', 105.44494066914405);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:10:00', 105.91518954803983);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:20:00', 105.17932691918186);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:30:00', 105.0265545565388);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:40:00', 105.72834323917131);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:50:00', 105.3212046616362);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:00:00', 105.25101430465428);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:10:00', 105.51898599885591);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:20:00', 105.88603103416416);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:30:00', 105.6764609467606);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:40:00', 105.9437335262162);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:50:00', 105.56486129098856);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:00:00', 105.44014210790266);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:10:00', 105.27107612729998);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:20:00', 110.12766042119884);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:30:00', 110.46097328333843);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:40:00', 110.09650709715108);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:50:00', 110.530961479503);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:00:00', 110.29088215265074);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:10:00', 110.03711684433249);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:20:00', 110.55993032748897);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:30:00', 110.59914205427324);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:40:00', 110.49597522798216);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:50:00', 110.0851886686103);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:00:00', 110.74491457116008);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:10:00', 110.33007256532116);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:20:00', 110.96297477536591);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:30:00', 110.24294858126936);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:40:00', 110.21683982012392);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:50:00', 110.64872681334936);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:00:00', 110.33724372312577);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:10:00', 110.57120569484695);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:20:00', 110.84902105905329);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:30:00', 110.07422340282801);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:40:00', 110.51798004935188);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:50:00', 110.35858103803595);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:00:00', 110.9330824329225);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:10:00', 110.75721595824172);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:20:00', 110.16966145686376);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:30:00', 110.6200419734301);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:40:00', 110.02639206216864);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:50:00', 110.71466266056092);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:00:00', 110.42276135702156);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:10:00', 110.64703918748785);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:20:00', 110.00938878239657);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:30:00', 110.74005330603822);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:40:00', 110.35879570234194);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:50:00', 110.27423905054137);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:00:00', 110.57262870011722);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:10:00', 110.01985893324671);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:20:00', 110.01651001138652);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:30:00', 110.6343588062614);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:40:00', 110.7691057682859);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:50:00', 110.854592886564);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:00:00', 110.49398499569142);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:10:00', 110.22574642265914);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:20:00', 110.1989620219235);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:30:00', 110.11844858740491);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:40:00', 110.85888099958702);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:50:00', 110.72004125251655);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:00:00', 110.24675835750195);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:10:00', 110.29249904823641);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:20:00', 110.07829374409832);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:30:00', 110.47142972006851);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:40:00', 110.61157527376456);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:50:00', 110.57876731472328);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:00:00', 110.74785940159668);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:10:00', 110.10603151988887);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:20:00', 110.40784334254565);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:30:00', 110.73885667679576);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:40:00', 110.98592930132185);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:50:00', 110.03159173929218);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:00:00', 110.44003663363205);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:10:00', 110.23601033453232);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:20:00', 110.38588089796673);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:30:00', 110.70774915007118);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:40:00', 110.80238457151769);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:50:00', 110.2695060563233);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:00:00', 110.43972332144956);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:10:00', 110.30751590489595);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:20:00', 110.72538811804337);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:30:00', 110.29258672963152);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:40:00', 110.0362065937212);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:50:00', 110.67543648315056);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:00:00', 110.9021817678074);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:10:00', 110.31611102580412);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:20:00', 110.07205126887513);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:30:00', 110.30667208476088);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:40:00', 110.2096486408832);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:50:00', 110.88528324901444);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:00:00', 110.51628474490116);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:10:00', 110.6566262439806);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:20:00', 110.6979105429185);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:30:00', 110.7173499344663);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:40:00', 110.13239608472007);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:50:00', 110.89353590114023);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:00:00', 110.09097519112751);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:10:00', 110.87044837143185);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:20:00', 110.65283407060114);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:30:00', 110.31014325206102);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:40:00', 110.02685067243759);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:50:00', 110.35220774702627);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:00:00', 110.00103050574343);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:10:00', 110.65596146610964);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:20:00', 110.08770330799145);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:30:00', 110.25361595070032);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:40:00', 110.42574750640628);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:50:00', 110.5237707383437);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:00:00', 110.56173231434866);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:10:00', 110.9703841087223);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:20:00', 110.27505286336691);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:30:00', 110.29340113028113);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:40:00', 110.32899197296658);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:50:00', 110.09626708999346);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:00:00', 115.92034509496644);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:10:00', 115.39922686791157);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:20:00', 115.05250578237221);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:30:00', 115.28072723909516);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:40:00', 115.50664108716694);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:50:00', 115.33329094145218);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:00:00', 115.33702847427412);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:10:00', 115.04981495129273);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:20:00', 115.05606091988008);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:30:00', 115.11740830894504);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:40:00', 115.4167621975373);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:50:00', 115.33128951639964);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:00:00', 115.62187401688368);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:10:00', 115.47647480832849);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:20:00', 115.4532001640554);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:30:00', 115.642196624224);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:40:00', 115.55298887565829);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:50:00', 115.49355051169212);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:00:00', 115.73059320995986);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:10:00', 115.76167513394451);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:20:00', 115.26306599022764);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:30:00', 115.51486681588571);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:40:00', 115.19562504740864);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:50:00', 115.17025430716672);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:00:00', 115.78013237676927);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:10:00', 115.03416311414273);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:20:00', 115.89882244794657);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:30:00', 115.81105607717902);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:40:00', 115.58443762971216);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:50:00', 115.26168713986871);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:00:00', 115.66526247866794);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:10:00', 115.82464119488087);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:20:00', 115.04056460532888);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:30:00', 115.98774330485566);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:40:00', 115.0104486595282);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:50:00', 115.96111537572357);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:00:00', 115.8245490214396);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:10:00', 115.38748286829578);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:20:00', 115.18520214522057);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:30:00', 115.3514485906698);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:40:00', 115.44665469221741);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:50:00', 115.55842156538937);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:00:00', 115.02560113526441);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:10:00', 115.87608037600435);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:20:00', 115.82420547970959);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:30:00', 115.01677905476782);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:40:00', 115.45409022327671);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:50:00', 115.12067421915043);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:00:00', 115.88727688865416);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:10:00', 115.52534722095196);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:20:00', 115.68163804179949);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:30:00', 115.539839218707);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:40:00', 115.22368930448073);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:50:00', 115.63567122140387);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:00:00', 115.60016350834304);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:10:00', 115.35732108719178);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:20:00', 115.74568200717638);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:30:00', 115.39704090173677);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:40:00', 115.61935157965172);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:50:00', 115.63610398605341);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:00:00', 115.89222340869463);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:10:00', 115.62902987838076);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:20:00', 115.52859321594143);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:30:00', 115.10513533273816);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:40:00', 115.88077983047351);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:50:00', 115.1130128018198);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:00:00', 115.98415202678764);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:10:00', 115.07473370436784);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:20:00', 115.3786367812187);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:30:00', 115.95605877330911);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:40:00', 115.89726054171273);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:50:00', 115.65894614508451);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:00:00', 115.24684375099228);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:10:00', 115.62974460698156);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:20:00', 115.69256150306074);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:30:00', 115.0166686892437);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:40:00', 115.3456336595683);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:50:00', 115.0234587345454);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:00:00', 115.97441992588934);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:10:00', 115.8448132541058);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:20:00', 115.65543028376051);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:30:00', 115.71208021448278);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:40:00', 115.4781607967377);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:50:00', 115.70510822555813);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:00:00', 115.52939212940808);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:10:00', 115.48561616524496);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:20:00', 115.52121044710016);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:30:00', 115.82674001086326);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:40:00', 115.20185675592418);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:50:00', 115.7595526052731);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:00:00', 115.40853540496208);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:10:00', 115.03500235867314);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:20:00', 115.13067419460987);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:30:00', 115.17327125512324);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:40:00', 115.64341194713968);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:50:00', 115.76641037060409);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:00:00', 115.83166136304564);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:10:00', 115.49761318146606);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:20:00', 115.45755696233412);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:30:00', 115.8410236957087);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:40:00', 120.82644489317774);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:50:00', 120.6549272826231);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:00:00', 120.08408232439287);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:10:00', 120.25385894689538);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:20:00', 120.831924185407);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:30:00', 120.80399119929008);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:40:00', 120.67437343753132);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:50:00', 120.56115061221644);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:00:00', 120.99023407744896);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:10:00', 120.65558185562443);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:20:00', 120.17956001975037);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:30:00', 120.72989876408664);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:40:00', 120.67663883423089);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:50:00', 120.98978608413451);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:00:00', 120.55715394264116);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:10:00', 120.3280233998929);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:20:00', 120.70094175757272);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:30:00', 120.98334063163055);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:40:00', 120.977634025921);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:50:00', 120.19680092048293);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:00:00', 120.06476432921309);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:10:00', 120.31070585738662);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:20:00', 120.40696681089787);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:30:00', 120.60145851705968);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:40:00', 120.00420037302388);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:50:00', 120.03280696643525);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:00:00', 120.37692640276228);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:10:00', 120.36665173229142);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:20:00', 120.94387167130972);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:30:00', 120.19997124628559);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:40:00', 120.05592686663337);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:50:00', 120.8548233151384);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:00:00', 120.02699959879585);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:10:00', 120.8519105228424);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:20:00', 120.03283338082785);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:30:00', 120.32936547420351);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:40:00', 120.80212574372938);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:50:00', 120.23921574911108);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:00:00', 120.75504438897896);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:10:00', 120.86207940009143);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:20:00', 120.20937931299572);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:30:00', 120.30095044093979);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:40:00', 120.5030615421094);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:50:00', 120.28598019189259);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:00:00', 120.10827568073782);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:10:00', 120.98699785663374);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:20:00', 120.96268312288082);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:30:00', 120.82371301225443);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:40:00', 120.61830046723459);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:50:00', 120.79977611813331);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:00:00', 120.57291856612294);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:10:00', 120.98601099526851);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:20:00', 120.03862837254096);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:30:00', 120.17184224024064);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:40:00', 120.80091055324695);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:50:00', 120.34000838523704);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:00:00', 120.60809451938822);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:10:00', 120.29035379289171);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:20:00', 120.9040104301264);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:30:00', 120.31136665078724);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:40:00', 120.1812663625463);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:50:00', 120.93438651835392);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:00:00', 120.94577682649124);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:10:00', 120.55026693362673);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:20:00', 120.49869839960226);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:30:00', 120.76419846529501);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:40:00', 120.32891781041769);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:50:00', 120.89725128563713);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:00:00', 120.81384751975108);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:10:00', 120.42830276437893);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:20:00', 120.76397470598722);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:30:00', 120.81497089297474);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:40:00', 120.32003571451868);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:50:00', 120.15467173662675);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:00:00', 120.06726059846304);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:10:00', 120.82311834144716);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:20:00', 120.1820536253275);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:30:00', 120.21635293924656);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:40:00', 120.51710522608116);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:50:00', 120.82906762171484);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:00:00', 120.52113940697737);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:10:00', 120.84858280524145);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:20:00', 120.96207479146118);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:30:00', 120.44426662820351);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:40:00', 120.90808339360385);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:50:00', 120.74990586115463);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:00:00', 120.67549253070091);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:10:00', 120.3298160130022);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:20:00', 120.88633713175562);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:30:00', 120.96457496772064);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:40:00', 120.98077386733937);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:50:00', 120.576124391448);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:00:00', 120.34659036767653);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:10:00', 120.5856561536969);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:20:00', 120.52944343859464);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:30:00', 120.36055981032291);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:40:00', 120.95179568270576);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:50:00', 120.33330010157015);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:00:00', 120.85482930960468);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:10:00', 120.79264328111007);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:20:00', 125.65287280517842);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:30:00', 125.64396553535299);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:40:00', 125.71734437088135);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:50:00', 125.42390450174223);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:00:00', 125.47850363605784);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:10:00', 125.26349082218289);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:20:00', 125.23066891402209);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:30:00', 125.29121932583885);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:40:00', 125.40797519911968);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:50:00', 125.0081113243922);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:00:00', 125.73769673323004);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:10:00', 125.55333399980495);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:20:00', 125.9338480235524);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:30:00', 125.584996803726);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:40:00', 125.91354166413394);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:50:00', 125.0099316793555);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:00:00', 125.58488234330359);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:10:00', 125.86727983852556);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:20:00', 125.61582159763391);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:30:00', 125.50454381881113);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:40:00', 125.52156499547934);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:50:00', 125.77681738674585);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:00:00', 125.329974313302);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:10:00', 125.37982398964373);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:20:00', 125.2000700117828);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:30:00', 125.76333038634014);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:40:00', 125.1751994132285);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:50:00', 125.83473591020085);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:00:00', 125.90173521962282);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:10:00', 125.81451848262671);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:20:00', 125.25740759463517);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:30:00', 125.26163239499365);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:40:00', 125.71705306683364);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:50:00', 125.81613305294701);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:00:00', 125.10330966729616);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:10:00', 125.60604692287441);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:20:00', 125.45815200785763);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:30:00', 125.08557013990955);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:40:00', 125.97649026685646);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:50:00', 125.01208175825936);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:00:00', 125.73601458974932);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:10:00', 125.56786793153258);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:20:00', 125.43308043543266);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:30:00', 125.77946478349509);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:40:00', 125.28416932149958);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:50:00', 125.1822931183685);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:00:00', 125.41011613343115);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:10:00', 125.16691492182012);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:20:00', 125.59900937371857);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:30:00', 125.4239352559672);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:40:00', 125.36411003724287);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:50:00', 125.71835636953129);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:00:00', 125.14126768660087);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:10:00', 125.15172467772771);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:20:00', 125.86074627485101);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:30:00', 125.59779947517357);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:40:00', 125.82445111331042);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:50:00', 125.13686466249423);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:00:00', 125.53973678755672);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:10:00', 125.24667751804859);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:20:00', 125.33573809756126);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:30:00', 125.4419433566887);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:40:00', 125.83330875165787);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:50:00', 125.22334573631028);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:00:00', 125.6414727679888);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:10:00', 125.55895516236252);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:20:00', 125.51127922981144);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:30:00', 125.24855436769563);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:40:00', 125.1715138490133);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:50:00', 125.90095334415271);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:00:00', 125.37706174808369);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:10:00', 125.72332978291209);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:20:00', 125.6247259341157);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:30:00', 125.77224995502243);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:40:00', 125.65948386671788);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:50:00', 125.94555285873936);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:00:00', 125.95422572214912);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:10:00', 125.8971838669447);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:20:00', 125.63645331661951);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:30:00', 125.92899146774526);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:40:00', 125.17840825816371);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:50:00', 125.31852650379227);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:00:00', 125.11065898281996);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:10:00', 125.61082397310557);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:20:00', 125.30540198466014);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:30:00', 125.84339428098431);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:40:00', 125.43905895299652);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:50:00', 125.23264337615092);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:00:00', 125.39144904762676);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:10:00', 125.47160455013405);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:20:00', 125.27470667067102);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:30:00', 125.1360922377444);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:40:00', 125.62329902316296);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:50:00', 125.66101845837649);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:00:00', 125.45546276762786);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:10:00', 125.43188406449724);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:20:00', 125.6870324820484);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:30:00', 125.6862262260098);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:40:00', 125.45986264549069);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:50:00', 125.87767184797652);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:00:00', 130.55752649889308);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:10:00', 130.47210457115082);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:20:00', 130.16083821591772);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:30:00', 130.0721132666919);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:40:00', 130.93182429388398);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:50:00', 130.88034006958418);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:00:00', 130.2960201404346);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:10:00', 130.81596206318963);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:20:00', 130.57712327914413);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:30:00', 130.9266257321872);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:40:00', 130.04945413971518);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:50:00', 130.2937922643273);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:00:00', 130.67992742849214);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:10:00', 130.52187784961671);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:20:00', 130.6815300385082);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:30:00', 130.1491898663459);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:40:00', 130.58692600595558);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:50:00', 130.59864771633175);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:00:00', 130.25205974444444);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:10:00', 130.51691670451189);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:20:00', 130.67374894414556);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:30:00', 130.8042564540647);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:40:00', 130.29362105587268);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:50:00', 130.77006018239035);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:00:00', 130.8588457916097);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:10:00', 130.31852233755862);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:20:00', 130.78765560716604);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:30:00', 130.54215704841198);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:40:00', 130.62851477591724);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:50:00', 130.72289371182964);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:00:00', 130.9918181823095);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:10:00', 130.42574240072523);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:20:00', 130.82852101016388);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:30:00', 130.4760645151284);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:40:00', 130.42018629271487);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:50:00', 130.1687714251814);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:00:00', 130.64233815481978);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:10:00', 130.49792908490352);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:20:00', 130.2525413864335);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:30:00', 130.77131974307093);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:40:00', 130.22991739697986);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:50:00', 130.91223647275896);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:00:00', 130.5269938909138);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:10:00', 130.22644406738803);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:20:00', 130.366184488456);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:30:00', 130.89817955425983);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:40:00', 130.0187264787575);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:50:00', 130.0602346252049);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:00:00', 130.08813722278666);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:10:00', 130.7376380926898);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:20:00', 130.11647754151107);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:30:00', 130.71971186904696);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:40:00', 130.26012816081476);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:50:00', 130.16294632396048);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:00:00', 130.89004024791146);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:10:00', 130.1610362610834);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:20:00', 130.76158826963186);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:30:00', 130.02155764039446);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:40:00', 130.06125995821742);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:50:00', 130.355675127596);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:00:00', 130.05219169492528);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:10:00', 130.92435792726826);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:20:00', 130.55939363652726);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:30:00', 130.94408625501342);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:40:00', 130.60133900877642);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:50:00', 130.5582953364659);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:00:00', 130.66243087111098);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:10:00', 130.1413215721195);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:20:00', 130.38149877843287);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:30:00', 130.3796352581514);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:40:00', 130.25363432472798);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:50:00', 130.43143986685902);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:00:00', 130.2389580537966);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:10:00', 130.66365461963042);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:20:00', 130.09582250337957);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:30:00', 130.8610290695083);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:40:00', 130.08579101152458);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:50:00', 130.9850413508669);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:00:00', 130.45040049483163);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:10:00', 130.96602251995662);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:20:00', 130.49534168094283);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:30:00', 130.69651652188273);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:40:00', 130.82244795778115);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:50:00', 130.23928203008177);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:00:00', 130.5342415952506);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:10:00', 130.86653257274222);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:20:00', 130.99315060435964);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:30:00', 130.70284782267896);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:40:00', 130.89953667800603);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:50:00', 130.59650905872653);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:00:00', 130.7439607481838);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:10:00', 130.73732494797144);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:20:00', 130.50496075815454);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:30:00', 130.86808847246445);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:40:00', 130.760322798989);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:50:00', 130.37022741605705);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:00:00', 130.8251499409934);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:10:00', 130.93879350865066);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:20:00', 130.42274793729106);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:30:00', 130.07145099626342);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:40:00', 135.5822275182252);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:50:00', 135.290875281548);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:00:00', 135.13915851315687);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:10:00', 135.01104907899665);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:20:00', 135.74063149714036);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:30:00', 135.9347280912333);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:40:00', 135.85710355684222);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:50:00', 135.77097325941608);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:00:00', 135.8455559646272);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:10:00', 135.89470738995027);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:20:00', 135.2489775061593);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:30:00', 135.3793364271599);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:40:00', 135.38449742742876);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:50:00', 135.03700395403507);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:00:00', 135.7464313179241);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:10:00', 135.04886193497214);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:20:00', 135.82877631810058);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:30:00', 135.91725559542292);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:40:00', 135.16359170124807);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:50:00', 135.269706845287);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:00:00', 135.75926140801715);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:10:00', 135.273780989814);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:20:00', 135.11920099470748);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:30:00', 135.6458249517251);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:40:00', 135.83506575523225);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:50:00', 135.1842346591327);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:00:00', 135.9559008321893);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:10:00', 135.71165753175615);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:20:00', 135.11854473865588);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:30:00', 135.56454284079038);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:40:00', 135.04791114699213);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:50:00', 135.66121579862698);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:00:00', 135.97693668813005);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:10:00', 135.01957571131328);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:20:00', 135.1784325684813);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:30:00', 135.9271176272339);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:40:00', 135.50456868397166);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:50:00', 135.18801922909378);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:00:00', 135.11824025540568);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:10:00', 135.12655456301354);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:20:00', 135.242658101112);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:30:00', 135.33625561865995);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:40:00', 135.5751045589858);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:50:00', 135.10703908836848);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:00:00', 135.49237126922029);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:10:00', 135.12773622383622);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:20:00', 135.48231168086377);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:30:00', 135.45502900632232);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:40:00', 135.31825332626258);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:50:00', 135.73105659175795);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:00:00', 135.25554367703347);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:10:00', 135.30102734148892);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:20:00', 135.1051556273559);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:30:00', 135.04007777331657);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:40:00', 135.77849168864321);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:50:00', 135.63694303307958);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:00:00', 135.74603303295677);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:10:00', 135.967451799846);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:20:00', 135.99283924105453);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:30:00', 135.38215001155572);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:40:00', 135.7203834940102);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:50:00', 135.72776660324354);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:00:00', 135.40006510083572);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:10:00', 135.18858626175364);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:20:00', 135.95836122705143);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:30:00', 135.25379951813704);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:40:00', 135.7276495111565);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:50:00', 135.49940105799575);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:00:00', 135.04478992920576);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:10:00', 135.16158472057415);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:20:00', 135.19104124900028);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:30:00', 135.1978962720263);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:40:00', 135.8797741927591);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:50:00', 135.42893695793265);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:00:00', 135.75721654223466);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:10:00', 135.1147303536879);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:20:00', 135.46564010969075);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:30:00', 135.28025985628759);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:40:00', 135.50777684605671);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:50:00', 135.85619380336865);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:00:00', 135.3321464077477);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:10:00', 135.76739484688176);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:20:00', 135.97454562186522);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:30:00', 135.23968953026392);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:40:00', 135.0013359421395);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:50:00', 135.14294928179038);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:00:00', 135.36118593983392);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:10:00', 135.01275592755974);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:20:00', 135.53935038333822);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:30:00', 135.06951475453644);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:40:00', 135.47701969383752);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:50:00', 135.49299753337226);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:00:00', 135.71005467665705);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:10:00', 135.6890138348129);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:20:00', 135.8286072166702);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:30:00', 135.7940921069671);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:40:00', 135.09662324353906);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:50:00', 135.01061425064736);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:00:00', 135.76969936568676);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:10:00', 135.02477095323124);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:20:00', 140.5524095544186);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:30:00', 140.38860659330004);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:40:00', 140.01734910815307);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:50:00', 140.1597221181258);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:00:00', 140.0314899124773);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:10:00', 140.1008617715402);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:20:00', 140.84218644510938);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:30:00', 140.95380320925605);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:40:00', 140.02786801928954);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:50:00', 140.68615634959215);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:00:00', 140.1455071617251);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:10:00', 140.66160049491523);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:20:00', 140.45608688522498);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:30:00', 140.45648268027682);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:40:00', 140.97569589085234);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:50:00', 140.67644041617714);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:00:00', 140.65864095209272);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:10:00', 140.52405153234466);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:20:00', 140.86515045160368);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:30:00', 140.66659420562374);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:40:00', 140.57133784911477);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:50:00', 140.16110513816938);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:00:00', 140.8304104307717);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:10:00', 140.78310964285572);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:20:00', 140.05597285732236);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:30:00', 140.70485364399892);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:40:00', 140.93107006701882);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:50:00', 140.57346515491787);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:00:00', 140.98225769949184);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:10:00', 140.32615795074054);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:20:00', 140.12590509899022);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:30:00', 140.70668039065674);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:40:00', 140.4140851064422);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:50:00', 140.03313073052286);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:00:00', 140.63351926697442);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:10:00', 140.1829796940263);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:20:00', 140.47967597934021);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:30:00', 140.34413142548922);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:40:00', 140.0426108415719);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:50:00', 140.08335049468613);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:00:00', 140.5809851312373);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:10:00', 140.30746319215652);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:20:00', 140.4437908140055);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:30:00', 140.02014453174584);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:40:00', 140.2128779109799);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:50:00', 140.5180134001954);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:00:00', 140.8832518623758);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:10:00', 140.72660708365876);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:20:00', 140.13260916673494);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:30:00', 140.6685779559208);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:40:00', 140.58626918907706);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:50:00', 140.76837168351068);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:00:00', 140.88943196987566);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:10:00', 140.3192286564333);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:20:00', 140.76723922879268);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:30:00', 140.5477150242652);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:40:00', 140.50047597246095);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:50:00', 140.59671771658958);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:00:00', 140.760457655149);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:10:00', 140.75917327120624);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:20:00', 140.01496604863922);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:30:00', 140.29081096369455);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:40:00', 140.67957024313557);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:50:00', 140.43814891956498);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:00:00', 140.47525383039314);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:10:00', 140.9418632322256);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:20:00', 140.76196779867192);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:30:00', 140.56609907176457);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:40:00', 140.85769612649491);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:50:00', 140.75518382700537);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:00:00', 140.01234263559613);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:10:00', 140.40596437354662);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:20:00', 140.48195068926188);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:30:00', 140.50113518688178);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:40:00', 140.2537437243218);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:50:00', 140.57588910299367);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:00:00', 140.5011731953);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:10:00', 140.32275275368934);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:20:00', 140.33672467424947);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:30:00', 140.6486065897093);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:40:00', 140.2221530753053);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:50:00', 140.28903592783888);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:00:00', 140.37179537480162);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:10:00', 140.50654885088002);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:20:00', 140.23551619652952);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:30:00', 140.5445764128223);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:40:00', 140.8115348719777);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:50:00', 140.13747874955092);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:00:00', 140.11103895370636);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:10:00', 140.77433273235442);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:20:00', 140.29246854553352);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:30:00', 140.25120631632427);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:40:00', 140.4406017165515);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:50:00', 140.58915042935652);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:00:00', 140.3146642494925);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:10:00', 140.28985323828846);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:20:00', 140.65486025792748);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:30:00', 140.05970824708018);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:40:00', 140.77085915678708);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:50:00', 140.22804689656098);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:00:00', 145.66987523875392);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:10:00', 145.1707350865351);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:20:00', 145.16940803645684);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:30:00', 145.28901088567568);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:40:00', 145.4980042071863);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:50:00', 145.2205579877147);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:00:00', 145.69597916914574);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:10:00', 145.58124546129085);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:20:00', 145.8574157476726);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:30:00', 145.6369219223971);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:40:00', 145.78766875330268);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:50:00', 145.35171586860537);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:00:00', 145.7281389158997);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:10:00', 145.30110340970037);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:20:00', 145.55264598747647);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:30:00', 145.60658471565296);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:40:00', 145.52161260399842);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:50:00', 145.96643519774298);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:00:00', 145.82570031315652);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:10:00', 145.25411620623473);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:20:00', 145.01084066406216);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:30:00', 145.765867810619);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:40:00', 145.89444272758473);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:50:00', 145.77793331580872);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:00:00', 145.44588175007783);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:10:00', 145.49194633436278);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:20:00', 145.30017295055927);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:30:00', 145.41543052738336);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:40:00', 145.8074991727321);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:50:00', 145.12258075325624);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:00:00', 145.72481690483806);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:10:00', 145.7129788820173);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:20:00', 145.2621944801272);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:30:00', 145.8563327369375);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:40:00', 145.11773145469112);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:50:00', 145.94891778385843);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:00:00', 145.34019456698357);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:10:00', 145.65932709875548);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:20:00', 145.43751530410034);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:30:00', 145.0593460373174);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:40:00', 145.88811740693234);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:50:00', 145.90368066651718);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:00:00', 145.95884728201386);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:10:00', 145.06150896281636);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:20:00', 145.8622261033162);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:30:00', 145.83755664275444);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:40:00', 145.43972995819828);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:50:00', 145.73981228170072);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:00:00', 145.12681461732356);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:10:00', 145.012754868371);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:20:00', 145.0505322143747);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:30:00', 145.88282679938504);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:40:00', 145.00081151155172);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:50:00', 145.43198725370831);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:00:00', 145.82429510444118);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:10:00', 145.9498559447054);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:20:00', 145.97480554095102);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:30:00', 145.4446798624002);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:40:00', 145.2216477801595);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:50:00', 145.37234684968098);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:00:00', 145.0455193928619);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:10:00', 145.57137581029124);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:20:00', 145.97361696560858);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:30:00', 145.415751650966);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:40:00', 145.08356782962744);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:50:00', 145.88023605798838);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:00:00', 145.40523929500432);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:10:00', 145.3418164246789);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:20:00', 145.59691692201514);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:30:00', 145.4793346319479);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:40:00', 145.64966501406303);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:50:00', 145.50216746294907);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:00:00', 145.91907700356387);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:10:00', 145.87403548263796);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:20:00', 145.46946986838512);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:30:00', 145.4998155317617);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:40:00', 145.27447045101775);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:50:00', 145.52002992975224);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:00:00', 145.91042830526916);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:10:00', 145.5409829527971);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:20:00', 145.63202944838724);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:30:00', 145.5344594614299);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:40:00', 145.41555312632602);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:50:00', 145.13658569771513);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:00:00', 145.8566582624446);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:10:00', 145.81325233346422);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:20:00', 145.78274734773234);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:30:00', 145.0640730008673);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:40:00', 145.80812656708284);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:50:00', 145.87468388022162);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:00:00', 145.67072629238953);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:10:00', 145.88096610600363);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:20:00', 145.09472335198168);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:30:00', 145.6638566197089);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:40:00', 145.08440919685958);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:50:00', 145.44861743475232);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:00:00', 145.89047384558398);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:10:00', 145.27914766231697);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:20:00', 145.69446146340147);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:30:00', 145.93709895795772);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:40:00', 150.95389038965732);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:50:00', 150.4695271916118);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:00:00', 150.71143300502908);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:10:00', 150.06902295783496);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:20:00', 150.94789146885924);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:30:00', 150.02560775759667);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:40:00', 150.46498881080538);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:50:00', 150.0295339298049);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:00:00', 150.9363084671751);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:10:00', 150.49309095688454);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:20:00', 150.42234840916416);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:30:00', 150.60626264187394);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:40:00', 150.29938954041432);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:50:00', 150.40643772440583);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:00:00', 150.29224512392312);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:10:00', 150.2138297054897);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:20:00', 150.74630354105142);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:30:00', 150.6907079934565);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:40:00', 150.03038540811488);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:50:00', 150.21958513347758);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:00:00', 150.17823386638824);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:10:00', 150.50507478083375);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:20:00', 150.79508618235164);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:30:00', 150.13488288529962);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:40:00', 150.1130873537007);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:50:00', 150.43758652726063);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:00:00', 150.5954318958787);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:10:00', 150.63272477805202);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:20:00', 150.85457355613744);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:30:00', 150.21404408605937);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:40:00', 150.34826903440288);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:50:00', 150.1586167071405);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:00:00', 150.21425811542989);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:10:00', 150.0486196281555);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:20:00', 150.33931288773363);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:30:00', 150.1331451778826);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:40:00', 150.3179776855876);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:50:00', 150.6026645961371);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:00:00', 150.63000440845244);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:10:00', 150.26746307195071);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:20:00', 150.30597385822765);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:30:00', 150.71097930825889);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:40:00', 150.2119519323343);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:50:00', 150.95334503876776);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:00:00', 150.21115641274213);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:10:00', 150.4672349421686);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:20:00', 150.35080350412545);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:30:00', 150.4910943472531);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:40:00', 150.6225395747);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:50:00', 150.89177789608738);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:00:00', 150.5461716161229);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:10:00', 150.63226578869234);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:20:00', 150.85689962949857);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:30:00', 150.9448742016535);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:40:00', 150.92408268138675);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:50:00', 150.73421350377888);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:00:00', 150.3034589100252);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:10:00', 150.7147463476092);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:20:00', 150.4921060710854);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:30:00', 150.0829502967536);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:40:00', 150.81063367796378);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:50:00', 150.36401854193718);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:00:00', 150.35446335237458);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:10:00', 150.23659343990658);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:20:00', 150.9489373250949);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:30:00', 150.72168461362784);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:40:00', 150.98344616973552);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:50:00', 150.90325970412428);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:00:00', 150.6322081841063);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:10:00', 150.07288034629926);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:20:00', 150.2769904157787);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:30:00', 150.45689512291858);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:40:00', 150.8623556168554);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:50:00', 150.65527174054034);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:00:00', 150.4105151231106);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:10:00', 150.3992746961309);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:20:00', 150.2225099991635);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:30:00', 150.6253682325025);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:40:00', 150.15362327609586);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:50:00', 150.45290266139548);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:00:00', 150.80716539465936);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:10:00', 150.53582530927787);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:20:00', 150.03001506957418);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:30:00', 150.29875278657747);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:40:00', 150.0375390473902);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:50:00', 150.30909323458138);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:00:00', 150.9957959535933);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:10:00', 150.5915194444149);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:20:00', 150.37621528200074);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:30:00', 150.93192625156226);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:40:00', 150.40413833750725);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:50:00', 150.49369775455642);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:00:00', 150.17480087886327);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:10:00', 150.30585649477317);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:20:00', 150.44827829973516);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:30:00', 150.1010045943311);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:40:00', 150.778458080293);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:50:00', 150.91295092252244);
+INSERT INTO ts_data (date, value) VALUES ('2016-12-20 17:00:00', 150.09800267705262);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 17:10:00', 150.28707660560792);
+
+
+DROP TABLE IF EXISTS prophet_model;
+SELECT MADLIB_SCHEMA.prophet_fit('ts_data', 'prophet_model', $${'ds':'date', 'y': 'value'}$$, $${'growth': 'linear' , 'changepoints': ['2016-11-30 00:20:00', '2016-11-30 05:40:00']}$$, True);
+SELECT ASSERT(count(*) = 1, 'prophet: prophet parameter counts error') FROM prophet_model;
+
+DROP TABLE IF EXISTS prophet_output;
+SELECT MADLIB_SCHEMA.prophet_predict('prophet_model', 'prophet_output', 1008, '10T', NULL, NULL, False);
+
diff --git a/src/ports/postgres/modules/facebook_prophet/test/prophet_test2.sql_in b/src/ports/postgres/modules/facebook_prophet/test/prophet_test2.sql_in
new file mode 100644
index 0000000..3d02aa2
--- /dev/null
+++ b/src/ports/postgres/modules/facebook_prophet/test/prophet_test2.sql_in
@@ -0,0 +1,99 @@
+/*
+* Test case of facebook-prophet
+*/
+
+m4_include(`SQLCommon.m4')
+
+select MADLIB_SCHEMA.prophet_fit();
+select MADLIB_SCHEMA.prophet_fit('usage');
+
+DROP TABLE IF EXISTS ts_data2;
+CREATE TABLE ts_data2 (date date, value float);
+
+
+INSERT INTO ts_data2 (date, value) VALUES ('1949-01-01', 112);INSERT INTO ts_data2 (date, value) VALUES ('1949-02-01', 118);
+INSERT INTO ts_data2 (date, value) VALUES ('1949-03-01', 132);INSERT INTO ts_data2 (date, value) VALUES ('1949-04-01', 129);
+INSERT INTO ts_data2 (date, value) VALUES ('1949-05-01', 121);INSERT INTO ts_data2 (date, value) VALUES ('1949-06-01', 135);
+INSERT INTO ts_data2 (date, value) VALUES ('1949-07-01', 148);INSERT INTO ts_data2 (date, value) VALUES ('1949-08-01', 148);
+INSERT INTO ts_data2 (date, value) VALUES ('1949-09-01', 136);INSERT INTO ts_data2 (date, value) VALUES ('1949-10-01', 119);
+INSERT INTO ts_data2 (date, value) VALUES ('1949-11-01', 104);INSERT INTO ts_data2 (date, value) VALUES ('1949-12-01', 118);
+INSERT INTO ts_data2 (date, value) VALUES ('1950-01-01', 115);INSERT INTO ts_data2 (date, value) VALUES ('1950-02-01', 126);
+INSERT INTO ts_data2 (date, value) VALUES ('1950-03-01', 141);INSERT INTO ts_data2 (date, value) VALUES ('1950-04-01', 135);
+INSERT INTO ts_data2 (date, value) VALUES ('1950-05-01', 125);INSERT INTO ts_data2 (date, value) VALUES ('1950-06-01', 149);
+INSERT INTO ts_data2 (date, value) VALUES ('1950-07-01', 170);INSERT INTO ts_data2 (date, value) VALUES ('1950-08-01', 170);
+INSERT INTO ts_data2 (date, value) VALUES ('1950-09-01', 158);INSERT INTO ts_data2 (date, value) VALUES ('1950-10-01', 133);
+INSERT INTO ts_data2 (date, value) VALUES ('1950-11-01', 114);INSERT INTO ts_data2 (date, value) VALUES ('1950-12-01', 140);
+INSERT INTO ts_data2 (date, value) VALUES ('1951-01-01', 145);INSERT INTO ts_data2 (date, value) VALUES ('1951-02-01', 150);
+INSERT INTO ts_data2 (date, value) VALUES ('1951-03-01', 178);INSERT INTO ts_data2 (date, value) VALUES ('1951-04-01', 163);
+INSERT INTO ts_data2 (date, value) VALUES ('1951-05-01', 172);INSERT INTO ts_data2 (date, value) VALUES ('1951-06-01', 178);
+INSERT INTO ts_data2 (date, value) VALUES ('1951-07-01', 199);INSERT INTO ts_data2 (date, value) VALUES ('1951-08-01', 199);
+INSERT INTO ts_data2 (date, value) VALUES ('1951-09-01', 184);INSERT INTO ts_data2 (date, value) VALUES ('1951-10-01', 162);
+INSERT INTO ts_data2 (date, value) VALUES ('1951-11-01', 146);INSERT INTO ts_data2 (date, value) VALUES ('1951-12-01', 166);
+INSERT INTO ts_data2 (date, value) VALUES ('1952-01-01', 171);INSERT INTO ts_data2 (date, value) VALUES ('1952-02-01', 180);
+INSERT INTO ts_data2 (date, value) VALUES ('1952-03-01', 193);INSERT INTO ts_data2 (date, value) VALUES ('1952-04-01', 181);
+INSERT INTO ts_data2 (date, value) VALUES ('1952-05-01', 183);INSERT INTO ts_data2 (date, value) VALUES ('1952-06-01', 218);
+INSERT INTO ts_data2 (date, value) VALUES ('1952-07-01', 230);INSERT INTO ts_data2 (date, value) VALUES ('1952-08-01', 242);
+INSERT INTO ts_data2 (date, value) VALUES ('1952-09-01', 209);INSERT INTO ts_data2 (date, value) VALUES ('1952-10-01', 191);
+INSERT INTO ts_data2 (date, value) VALUES ('1952-11-01', 172);INSERT INTO ts_data2 (date, value) VALUES ('1952-12-01', 194);
+INSERT INTO ts_data2 (date, value) VALUES ('1953-01-01', 196);INSERT INTO ts_data2 (date, value) VALUES ('1953-02-01', 196);
+INSERT INTO ts_data2 (date, value) VALUES ('1953-03-01', 236);INSERT INTO ts_data2 (date, value) VALUES ('1953-04-01', 235);
+INSERT INTO ts_data2 (date, value) VALUES ('1953-05-01', 229);INSERT INTO ts_data2 (date, value) VALUES ('1953-06-01', 243);
+INSERT INTO ts_data2 (date, value) VALUES ('1953-07-01', 264);INSERT INTO ts_data2 (date, value) VALUES ('1953-08-01', 272);
+INSERT INTO ts_data2 (date, value) VALUES ('1953-09-01', 237);INSERT INTO ts_data2 (date, value) VALUES ('1953-10-01', 211);
+INSERT INTO ts_data2 (date, value) VALUES ('1953-11-01', 180);INSERT INTO ts_data2 (date, value) VALUES ('1953-12-01', 201);
+INSERT INTO ts_data2 (date, value) VALUES ('1954-01-01', 204);INSERT INTO ts_data2 (date, value) VALUES ('1954-02-01', 188);
+INSERT INTO ts_data2 (date, value) VALUES ('1954-03-01', 235);INSERT INTO ts_data2 (date, value) VALUES ('1954-04-01', 227);
+INSERT INTO ts_data2 (date, value) VALUES ('1954-05-01', 234);INSERT INTO ts_data2 (date, value) VALUES ('1954-06-01', 264);
+INSERT INTO ts_data2 (date, value) VALUES ('1954-07-01', 302);INSERT INTO ts_data2 (date, value) VALUES ('1954-08-01', 293);
+INSERT INTO ts_data2 (date, value) VALUES ('1954-09-01', 259);INSERT INTO ts_data2 (date, value) VALUES ('1954-10-01', 229);
+INSERT INTO ts_data2 (date, value) VALUES ('1954-11-01', 203);INSERT INTO ts_data2 (date, value) VALUES ('1954-12-01', 229);
+INSERT INTO ts_data2 (date, value) VALUES ('1955-01-01', 242);INSERT INTO ts_data2 (date, value) VALUES ('1955-02-01', 233);
+INSERT INTO ts_data2 (date, value) VALUES ('1955-03-01', 267);INSERT INTO ts_data2 (date, value) VALUES ('1955-04-01', 269);
+INSERT INTO ts_data2 (date, value) VALUES ('1955-05-01', 270);INSERT INTO ts_data2 (date, value) VALUES ('1955-06-01', 315);
+INSERT INTO ts_data2 (date, value) VALUES ('1955-07-01', 364);INSERT INTO ts_data2 (date, value) VALUES ('1955-08-01', 347);
+INSERT INTO ts_data2 (date, value) VALUES ('1955-09-01', 312);INSERT INTO ts_data2 (date, value) VALUES ('1955-10-01', 274);
+INSERT INTO ts_data2 (date, value) VALUES ('1955-11-01', 237);INSERT INTO ts_data2 (date, value) VALUES ('1955-12-01', 278);
+INSERT INTO ts_data2 (date, value) VALUES ('1956-01-01', 284);INSERT INTO ts_data2 (date, value) VALUES ('1956-02-01', 277);
+INSERT INTO ts_data2 (date, value) VALUES ('1956-03-01', 317);INSERT INTO ts_data2 (date, value) VALUES ('1956-04-01', 313);
+INSERT INTO ts_data2 (date, value) VALUES ('1956-05-01', 318);INSERT INTO ts_data2 (date, value) VALUES ('1956-06-01', 374);
+INSERT INTO ts_data2 (date, value) VALUES ('1956-07-01', 413);INSERT INTO ts_data2 (date, value) VALUES ('1956-08-01', 405);
+INSERT INTO ts_data2 (date, value) VALUES ('1956-09-01', 355);INSERT INTO ts_data2 (date, value) VALUES ('1956-10-01', 306);
+INSERT INTO ts_data2 (date, value) VALUES ('1956-11-01', 271);INSERT INTO ts_data2 (date, value) VALUES ('1956-12-01', 306);
+INSERT INTO ts_data2 (date, value) VALUES ('1957-01-01', 315);INSERT INTO ts_data2 (date, value) VALUES ('1957-02-01', 301);
+INSERT INTO ts_data2 (date, value) VALUES ('1957-03-01', 356);INSERT INTO ts_data2 (date, value) VALUES ('1957-04-01', 348);
+INSERT INTO ts_data2 (date, value) VALUES ('1957-05-01', 355);INSERT INTO ts_data2 (date, value) VALUES ('1957-06-01', 422);
+INSERT INTO ts_data2 (date, value) VALUES ('1957-07-01', 465);INSERT INTO ts_data2 (date, value) VALUES ('1957-08-01', 467);
+INSERT INTO ts_data2 (date, value) VALUES ('1957-09-01', 404);INSERT INTO ts_data2 (date, value) VALUES ('1957-10-01', 347);
+INSERT INTO ts_data2 (date, value) VALUES ('1957-11-01', 305);INSERT INTO ts_data2 (date, value) VALUES ('1957-12-01', 336);
+INSERT INTO ts_data2 (date, value) VALUES ('1958-01-01', 340);INSERT INTO ts_data2 (date, value) VALUES ('1958-02-01', 318);
+INSERT INTO ts_data2 (date, value) VALUES ('1958-03-01', 362);INSERT INTO ts_data2 (date, value) VALUES ('1958-04-01', 348);
+INSERT INTO ts_data2 (date, value) VALUES ('1958-05-01', 363);INSERT INTO ts_data2 (date, value) VALUES ('1958-06-01', 435);
+INSERT INTO ts_data2 (date, value) VALUES ('1958-07-01', 491);INSERT INTO ts_data2 (date, value) VALUES ('1958-08-01', 505);
+INSERT INTO ts_data2 (date, value) VALUES ('1958-09-01', 404);INSERT INTO ts_data2 (date, value) VALUES ('1958-10-01', 359);
+INSERT INTO ts_data2 (date, value) VALUES ('1958-11-01', 310);INSERT INTO ts_data2 (date, value) VALUES ('1958-12-01', 337);
+INSERT INTO ts_data2 (date, value) VALUES ('1959-01-01', 360);INSERT INTO ts_data2 (date, value) VALUES ('1959-02-01', 342);
+INSERT INTO ts_data2 (date, value) VALUES ('1959-03-01', 406);INSERT INTO ts_data2 (date, value) VALUES ('1959-04-01', 396);
+INSERT INTO ts_data2 (date, value) VALUES ('1959-05-01', 420);INSERT INTO ts_data2 (date, value) VALUES ('1959-06-01', 472);
+INSERT INTO ts_data2 (date, value) VALUES ('1959-07-01', 548);INSERT INTO ts_data2 (date, value) VALUES ('1959-08-01', 559);
+INSERT INTO ts_data2 (date, value) VALUES ('1959-09-01', 463);INSERT INTO ts_data2 (date, value) VALUES ('1959-10-01', 407);
+INSERT INTO ts_data2 (date, value) VALUES ('1959-11-01', 362);INSERT INTO ts_data2 (date, value) VALUES ('1959-12-01', 405);
+INSERT INTO ts_data2 (date, value) VALUES ('1960-01-01', 417);INSERT INTO ts_data2 (date, value) VALUES ('1960-02-01', 391);
+INSERT INTO ts_data2 (date, value) VALUES ('1960-03-01', 419);INSERT INTO ts_data2 (date, value) VALUES ('1960-04-01', 461);
+INSERT INTO ts_data2 (date, value) VALUES ('1960-05-01', 472);INSERT INTO ts_data2 (date, value) VALUES ('1960-06-01', 535);
+INSERT INTO ts_data2 (date, value) VALUES ('1960-07-01', 622);INSERT INTO ts_data2 (date, value) VALUES ('1960-08-01', 606);
+INSERT INTO ts_data2 (date, value) VALUES ('1960-09-01', 508);INSERT INTO ts_data2 (date, value) VALUES ('1960-10-01', 461);
+INSERT INTO ts_data2 (date, value) VALUES ('1960-11-01', 390);INSERT INTO ts_data2 (date, value) VALUES ('1960-12-01', 432);
+
+
+ALTER TABLE ts_data2 ADD COLUMN cap FLOAT;
+ALTER TABLE ts_data2 ADD COLUMN floor FLOAT;
+UPDATE ts_data2 SET cap=600;
+UPDATE ts_data2 SET floor=0;
+
+DROP TABLE IF EXISTS prophet_model2;
+SELECT MADLIB_SCHEMA.prophet_fit('ts_data2', 'prophet_model2', $${'ds':'date', 'y': 'value', 'cap': 'cap', 'floor': 'floor'}$$, $${'growth': 'logistic'}$$, True);
+SELECT ASSERT(count(*) = 1, 'prophet: prophet parameter counts error') FROM prophet_model2;
+
+DROP TABLE IF EXISTS prophet_output;
+SELECT MADLIB_SCHEMA.prophet_predict('prophet_model2', 'prophet_output2', 10, '1D', 600, 0, False);
+
diff --git a/src/ports/postgres/modules/gbdt/__init__.py_in b/src/ports/postgres/modules/gbdt/__init__.py_in
new file mode 100644
index 0000000..e69de29
diff --git a/src/ports/postgres/modules/gbdt/gbdt.py_in b/src/ports/postgres/modules/gbdt/gbdt.py_in
new file mode 100644
index 0000000..4004c65
--- /dev/null
+++ b/src/ports/postgres/modules/gbdt/gbdt.py_in
@@ -0,0 +1,306 @@
+# -*- coding:utf-8 -*-
+# Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+#
+# openGauss is licensed under Mulan PSL v2.
+# You can use this software according to the terms
+# and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+# http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ----------------------------------------------------------------------------
+
+import numpy as np
+import plpy
+from math import sqrt, ceil
+
+from utilities.control import MinWarning
+from utilities.utilities import _assert
+from utilities.utilities import extract_keyvalue_params
+from utilities.validate_args import get_expr_type
+
+from recursive_partitioning.decision_tree import _tree_train_using_bins
+from recursive_partitioning.decision_tree import _get_features_to_use
+from recursive_partitioning.decision_tree import _classify_features
+from recursive_partitioning.decision_tree import _get_filter_str
+from recursive_partitioning.decision_tree import _get_bins
+from recursive_partitioning.decision_tree import get_feature_str
+
+
+
+def gdbt_mean_square_error(y_pred, y):
+ y_pred = np.reshape(y_pred,[-1])
+ y = np.reshape(y,[-1])
+ return 0.5 * np.square(y_pred - y) / y.shape[0]
+
+def create_model_table(schema_madlib,output_table_name,tree_state, bins, tree_iter):
+ head = ''
+ if tree_iter == 0:
+ head = """CREATE TABLE {output_table_name} AS""".format(output_table_name = output_table_name)
+ else:
+ head = "INSERT INTO " + output_table_name
+
+ depth = (tree_state['pruned_depth'] if 'pruned_depth' in tree_state
+ else tree_state['tree_depth'])
+
+ sql = head + """ SELECT {tree_iter} as iteration,
+ $1 AS tree,
+ $2 as cat_levels_in_text,
+ $3 as cat_n_levels,
+ {depth} as tree_depth
+ """.format(tree_iter = tree_iter, depth=depth)
+
+ sql_plan = plpy.prepare(sql, ['{0}.bytea8'.format(schema_madlib), 'text[]','integer[]'])
+ plpy.execute(sql_plan,[tree_state['tree_state'],bins['cat_origin'],bins['cat_n']])
+ return None
+
+def create_summary_table(output_table, null_proxy, cat_f, con_f, learning_rate, is_classification, predict_dt_prob, num_trees, source_table):
+ summary_table_name = output_table + '_summary'
+ null_proxy_str="NULL" if null_proxy is None else "'{0}'".format(null_proxy)
+ cat_f = ','.join(cat_f)
+ con_f = ','.join(con_f)
+ create_sql = """CREATE TABLE {summary_table_name} AS SELECT
+ 'GBDT'::TEXT as method,
+ '{cat_f}'::text AS cat_features,
+ '{con_f}'::text AS con_features,
+ '{source_table}'::text AS source_table,
+ '{output_table}'::text AS model_table,
+ {null_proxy_str}::text AS null_proxy,
+ {learning_rate}::double precision AS learning_rate,
+ {is_classification}::BOOLEAN AS is_classification,
+ '{predict_dt_prob}'::text AS predict_dt_prob,
+ {num_trees}::INTEGER AS num_trees
+ """.format(**locals())
+ plpy.execute(create_sql)
+
+def create_test_table(test_result_table, id, y_pred):
+ plpy.execute( """CREATE TABLE {test_result_table} AS SELECT
+ unnest(ARRAY{id}) as id,
+ unnest(ARRAY{y_pred}) as test_prediction
+ """.format(**locals()))
+
+def tree_fit(schema_madlib, bins, training_table_name, cat_features, con_features,
+ boolean_cats, num_bins,weights,
+ dep, min_split, min_bucket,
+ max_tree_depth, filter_null, dep_n_levels,
+ split_criterion,
+ num_random_features,
+ max_n_surr,null_proxy,verbose_mode=False,**kwargs):
+
+ msg_level = "notice" if verbose_mode else "warning"
+
+ with MinWarning(msg_level):
+ tree = _tree_train_using_bins(
+ schema_madlib, bins, training_table_name, cat_features, con_features,
+ boolean_cats, num_bins, weights, dep, min_split,
+ min_bucket, max_tree_depth, filter_null, dep_n_levels,
+ False, split_criterion, True,
+ num_random_features, max_n_surr, null_proxy)
+
+ tree['grp_key'] = ' '
+ return tree
+
+def tree_gradient_predict(schema_madlib, model_table,iteration, id_col_name, cat_features_str,
+ con_features_str, is_classification, predict_dt_prob, source_table):
+ id = []
+ res = []
+
+ if not is_classification or (predict_dt_prob == "response"):
+ sql = """ SELECT
+ {id_col_name} as id,
+ {schema_madlib}._predict_dt_response(
+ tree,
+ {cat_features_str}::INTEGER[],
+ {con_features_str}::DOUBLE PRECISION[]) as gradient_prediction
+ FROM (
+ SELECT * FROM {model_table}
+ WHERE iteration = {iteration}) as m,
+ {source_table}""".format(**locals())
+ else :
+ sql = """ SELECT
+ {id_col_name} as id,
+ {schema_madlib}._predict_dt_prob(
+ tree,
+ {cat_features_str}::INTEGER[],
+ {con_features_str}::DOUBLE PRECISION[]) as gradient_prediction
+ FROM (
+ SELECT * FROM {model_table}
+ WHERE iteration = {iteration}) as m,
+ {source_table}""".format(**locals())
+ rows = plpy.execute(sql)
+ for r in rows:
+ id.append(r['id'])
+ res.append(r['gradient_prediction'])
+ return id , res
+
+def _get_gradient(y, y_pred, id, iteration, y_table_name):
+ # 1.calculate the loss
+ compute_loss = gdbt_mean_square_error(y_pred, y)
+
+ # 2.calculate the gradient
+ cal_gradient = (np.gradient(compute_loss)).tolist()
+
+ # 3.add column called gradient to source_table
+ if iteration == 1:
+ addCol_sql = """alter table {y_table_name} add gradient double precision""".format(**locals())
+ plpy.execute(addCol_sql)
+
+ update_sql = """UPDATE {y_table_name} SET gradient = temp_gradient FROM (SELECT unnest(ARRAY{cal_gradient}) as temp_gradient,
+ unnest(ARRAY{id}) as temp_id)q WHERE id = temp_id""".format(**locals())
+ plpy.execute(update_sql)
+ return cal_gradient
+
+def gbdt_fit(schema_madlib,training_table_name, output_table_name,
+ id_col_name, dependent_variable, list_of_features,
+ list_of_features_to_exclude, weights,
+ num_trees, num_random_features,
+ max_tree_depth, min_split, min_bucket, num_bins,
+ null_handling_params, is_classification,
+ predict_dt_prob = None, learning_rate = None,
+ verbose=False, **kwargs):
+
+ # 1. default paremeters
+ num_trees = 100 if num_trees is None else num_trees
+ weights = '1' if not weights or not weights.strip() else weights.strip()
+ max_tree_depth = 10 if max_tree_depth is None else max_tree_depth
+ min_split = 20 if min_split is None and min_bucket is None else min_split
+ min_bucket = min_split // 3 if not min_bucket else min_bucket
+ min_split = min_bucket * 3 if not min_split else min_split
+ num_bins = 100 if num_bins is None else num_bins
+ learning_rate = 0.01 if learning_rate is None else learning_rate
+ if predict_dt_prob not in ["response", "prob", None]:
+ plpy.error("""
+ invalid choice: '{predict_dt_prob}' (choose from 'response', 'prob')
+ """.format(predict_dt_prob= predict_dt_prob))
+ predict_dt_prob = "response" if predict_dt_prob is None else predict_dt_prob
+
+ null_handling_dict = extract_keyvalue_params(
+ null_handling_params,
+ dict(max_surrogates=int, null_as_category=bool),
+ dict(max_surrogates=0, null_as_category=False))
+ max_n_surr = null_handling_dict['max_surrogates']
+ null_as_category = null_handling_dict['null_as_category']
+ null_proxy = "__NULL__" if null_as_category else None
+ if null_as_category:
+ # can't have two ways of handling tuples with NULL values
+ max_n_surr = 0
+ _assert(max_n_surr >= 0,
+ "Maximum number of surrogates ({0}) should be non-negative".
+ format(max_n_surr))
+
+ # preprocess arguments
+ # expand "*" syntax and exclude some features
+ features = _get_features_to_use(schema_madlib,
+ training_table_name, list_of_features, list_of_features_to_exclude,
+ id_col_name, '1', dependent_variable, None)
+
+ split_criterion = 'mse'
+ if num_random_features is None:
+ n_all_features = len(features)
+ num_random_features = int(sqrt(n_all_features) if is_classification else ceil(float(n_all_features) / 3))
+
+ _assert(0 < num_random_features <= len(features),
+ "GBDT error: Number of features to be selected "
+ "is more than the actual number of features.")
+
+ all_cols_types = dict([(f, get_expr_type(f, training_table_name))
+ for f in features])
+ cat_features, ordered_cat_features, boolean_cats, con_features = \
+ _classify_features(all_cols_types, features)
+
+
+ filter_null = _get_filter_str(dependent_variable, None)
+
+ n_rows = plpy.execute("SELECT count(*) FROM {0} WHERE {1}".format(training_table_name, filter_null))[0]['count']
+ dep_n_levels = 1
+
+ bins = _get_bins(schema_madlib, training_table_name,
+ cat_features, ordered_cat_features,
+ con_features, num_bins, dependent_variable,
+ boolean_cats, n_rows, False,
+ dep_n_levels, filter_null, null_proxy)
+ bins['grp_key_cat'] = ['']
+ cat_features = bins['cat_features']
+
+ tree = tree_fit(schema_madlib, bins, training_table_name, cat_features, con_features,
+ boolean_cats, num_bins,weights,
+ dependent_variable, min_split, min_bucket,
+ max_tree_depth, filter_null, dep_n_levels,
+ split_criterion,
+ num_random_features,
+ max_n_surr,null_proxy)
+ create_model_table(schema_madlib, output_table_name,tree, bins, 0)
+
+ cat_features_str, con_features_str = get_feature_str(schema_madlib,
+ training_table_name,
+ cat_features,
+ con_features,
+ "m.cat_levels_in_text", "m.cat_n_levels",
+ null_proxy)
+
+ id, y_pred = tree_gradient_predict(schema_madlib,output_table_name,0, id_col_name, cat_features_str,
+ con_features_str,is_classification,predict_dt_prob, training_table_name)
+
+ y_res = plpy.execute("""SELECT {dependent_variable} AS y FROM {training_table_name}""".format(**locals()))
+ y = []
+ for row in y_res:
+ y.append(row['y'])
+
+ for sample_id in range(1, num_trees):
+ gradient = _get_gradient(y , y_pred, id, sample_id, training_table_name)
+
+ tree = tree_fit (schema_madlib, bins, training_table_name, cat_features, con_features,
+ boolean_cats, num_bins, weights,
+ 'gradient', min_split, min_bucket,
+ max_tree_depth, filter_null, dep_n_levels,
+ split_criterion,
+ num_random_features,
+ max_n_surr,null_proxy)
+
+ create_model_table(schema_madlib, output_table_name,tree, bins, sample_id)
+ id, y_temp = tree_gradient_predict(schema_madlib,output_table_name,sample_id, id_col_name, cat_features_str,
+ con_features_str,is_classification, predict_dt_prob,training_table_name)
+ y_pred -= np.multiply(learning_rate, y_temp)
+
+ plpy.execute("""ALTER TABLE {training_table_name} DROP COLUMN IF EXISTS gradient CASCADE
+ """.format(training_table_name=training_table_name))
+
+ create_summary_table(output_table_name, null_proxy, bins['cat_features'],
+ bins['con_features'], learning_rate, is_classification, predict_dt_prob,
+ num_trees, training_table_name)
+
+
+
+def gbdt_predict(schema_madlib, test_table_name, model_table_name, output_table_name, id_col_name, **kwargs):
+
+ num_tree = plpy.execute("""SELECT COUNT(*) AS count FROM {model_table_name}""".format(**locals()))[0]['count']
+ if num_tree == 0:
+ plpy.error("The GBDT-method has no trees")
+
+
+ elements = plpy.execute("""SELECT * FROM {model_table_name}_summary""".format(**locals()))[0]
+ cat_features = [] if not elements['cat_features'] else elements['cat_features'].split(',')
+ con_features = [] if not elements['con_features'] else elements['con_features'].split(',')
+ null_proxy = elements['null_proxy']
+ learning_rate = elements['learning_rate']
+ is_classification = elements['is_classification']
+ predict_dt_prob = elements['predict_dt_prob']
+
+ cat_features_str, con_features_str = get_feature_str(
+ schema_madlib, test_table_name, cat_features, con_features, "m.cat_levels_in_text", "m.cat_n_levels", null_proxy)
+
+ id, y_pred = tree_gradient_predict(schema_madlib,model_table_name ,0, id_col_name, cat_features_str,
+ con_features_str,is_classification,predict_dt_prob,test_table_name)
+
+ for i in range(1, num_tree):
+ _, gradient = tree_gradient_predict(schema_madlib,model_table_name , i, id_col_name, cat_features_str,
+ con_features_str,is_classification,predict_dt_prob,test_table_name)
+ y_pred -= np.multiply(learning_rate, gradient)
+ if is_classification:
+ y_pred = np.around(y_pred)
+ create_test_table(output_table_name, id, y_pred.tolist())
\ No newline at end of file
diff --git a/src/ports/postgres/modules/gbdt/gbdt.sql_in b/src/ports/postgres/modules/gbdt/gbdt.sql_in
new file mode 100644
index 0000000..70da99c
--- /dev/null
+++ b/src/ports/postgres/modules/gbdt/gbdt.sql_in
@@ -0,0 +1,332 @@
+/*
+ * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+ *
+ * openGauss is licensed under Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *
+ * http://license.coscl.org.cn/MulanPSL2
+ *
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * -------------------------------------------------------------------------
+ */
+
+m4_include(`SQLCommon.m4')
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER,
+ num_bins INTEGER,
+ null_handling_params TEXT,
+ is_classification BOOLEAN,
+ predict_dt_prob TEXT,
+ learning_rate DOUBLE PRECISION,
+ verbose BOOLEAN,
+ sample_ratio DOUBLE PRECISION
+)
+RETURNS VOID AS $$
+PythonFunction(gbdt, gbdt, gbdt_fit)
+$$ LANGUAGE plpythonu VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER,
+ num_bins INTEGER,
+ null_handling_params TEXT,
+ is_classification BOOLEAN,
+ predict_dt_prob TEXT,
+ learning_rate DOUBLE PRECISION,
+ verbose BOOLEAN
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, $13, $14, $15, $16, $17, $18, NULL);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER,
+ num_bins INTEGER,
+ null_handling_params TEXT,
+ is_classification BOOLEAN,
+ predict_dt_prob TEXT,
+ learning_rate DOUBLE PRECISION
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, $13, $14, $15, $16, $17, FALSE::BOOLEAN);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER,
+ num_bins INTEGER,
+ null_handling_params TEXT,
+ is_classification BOOLEAN,
+ predict_dt_prob TEXT
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, $13, $14, $15, $16, NULL);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER,
+ num_bins INTEGER,
+ null_handling_params TEXT,
+ is_classification BOOLEAN
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, $13, $14, $15, NULL);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER,
+ num_bins INTEGER,
+ null_handling_params TEXT
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, $13, $14, FALSE);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER,
+ num_bins INTEGER
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, $13, NULL::TEXT);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER,
+ min_bucket INTEGER
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, 100::INTEGER);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER,
+ min_split INTEGER
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, ($11/3)::INTEGER);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER,
+ max_tree_depth INTEGER
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ 20::INTEGER);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER,
+ num_random_features INTEGER
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, 1::INTEGER);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT,
+ num_trees INTEGER
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, NULL::INTEGER);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT,
+ weights TEXT
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, 30::INTEGER);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT,
+ list_of_features_to_exclude TEXT
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, NULL::TEXT);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train(
+ training_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT,
+ dependent_variable TEXT,
+ list_of_features TEXT
+)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, NULL::TEXT);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_predict(
+ test_table_name TEXT,
+ model_table_name TEXT,
+ output_table_name TEXT,
+ id_col_name TEXT
+) RETURNS VOID AS $$
+PythonFunction(gbdt, gbdt, gbdt_predict)
+$$ LANGUAGE plpythonu VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
\ No newline at end of file
diff --git a/src/ports/postgres/modules/gbdt/test/gbdt.sql b/src/ports/postgres/modules/gbdt/test/gbdt.sql
new file mode 100644
index 0000000..1bdbd28
--- /dev/null
+++ b/src/ports/postgres/modules/gbdt/test/gbdt.sql
@@ -0,0 +1,59 @@
+DROP TABLE IF EXISTS dt_golf CASCADE;
+DROP TABLE IF EXISTS train_output,train_output_summary;
+CREATE TABLE dt_golf (
+ id integer NOT NULL,
+ "OUTLOOK" text,
+ temperature double precision,
+ humidity double precision,
+ "Cont_features" double precision[],
+ cat_features text[],
+ windy boolean,
+ class integer
+) m4_ifdef(`__COLUMNS__', `WITH(ORIENTATION=COLUMN)', `');
+
+INSERT INTO dt_golf (id,"OUTLOOK",temperature,humidity,"Cont_features",cat_features, windy,class) VALUES
+(1, 'sunny', 85, 85,ARRAY[85, 85], ARRAY['a', 'b'], false, 0),
+(2, 'sunny', 80, 90, ARRAY[80, 90], ARRAY['a', 'b'], true, 0),
+(3, 'overcast', 83, 78, ARRAY[83, 78], ARRAY['a', 'b'], false, 1),
+(4, 'rain', 70, NULL, ARRAY[70, 96], ARRAY['a', 'b'], false, 1),
+(5, 'rain', 68, 80, ARRAY[68, 80], ARRAY['a', 'b'], false, 1),
+(6, 'rain', NULL, 70, ARRAY[65, 70], ARRAY['a', 'b'], true, 0),
+(7, 'overcast', 64, 65, ARRAY[64, 65], ARRAY['c', 'b'], NULL , 1),
+(8, 'sunny', 72, 95, ARRAY[72, 95], ARRAY['a', 'b'], false, 0),
+(9, 'sunny', 69, 70, ARRAY[69, 70], ARRAY['a', 'b'], false, 1),
+(10, 'rain', 75, 80, ARRAY[75, 80], ARRAY['a', 'b'], false, 1),
+(11, 'sunny', 75, 70, ARRAY[75, 70], ARRAY['a', 'd'], true, 1),
+(12, 'overcast', 72, 90, ARRAY[72, 90], ARRAY['c', 'b'], NULL, 1),
+(13, 'overcast', 81, 75, ARRAY[81, 75], ARRAY['a', 'b'], false, 1),
+(15, NULL, 81, 75, ARRAY[81, 75], ARRAY['a', 'b'], false, 1),
+(16, 'overcast', NULL, 75, ARRAY[81, 75], ARRAY['a', 'd'], false,1),
+(14, 'rain', 71, 80, ARRAY[71, 80], ARRAY['c', 'b'], true, 0);
+
+-------------------------------------------------------------------------
+-- classification without grouping
+ select madlib.gbdt_train('dt_golf', -- source table
+ 'train_output', -- output model table
+ 'id' , -- id column
+ 'class', -- response
+ 'windy, "Cont_features"[1]', -- features
+ NULL, -- exclude columns
+ NULL,
+ NULL, -- no grouping
+ 30, -- num of trees
+ 2, -- num of random features
+ TRUE, -- importance
+ 1, -- num_permutations
+ 10, -- max depth
+ 1, -- min split
+ 1, -- min bucket
+ 8, -- number of bins per continuous variable
+ 'max_surrogates=0',
+ 'response',
+ 0.001,
+ FALSE,
+ 0.5
+);
+
+CREATE TABLE dt_golf2 as SELECT * from dt_golf;
+DROP TABLE IF EXISTS test_output;
+select madlib.gbdt_predict('dt_golf2','train_output','test_output','id');
diff --git a/src/ports/postgres/modules/glm/glm.sql_in b/src/ports/postgres/modules/glm/glm.sql_in
index 377dfea..4ebc6eb 100644
--- a/src/ports/postgres/modules/glm/glm.sql_in
+++ b/src/ports/postgres/modules/glm/glm.sql_in
@@ -649,7 +649,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_merge_states(
state2 MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_merge_states'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -658,7 +658,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_final(
state MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -671,7 +671,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_poisson_log_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_poisson_log_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -699,7 +699,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_poisson_identity_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_poisson_identity_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -727,7 +727,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_poisson_sqrt_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_poisson_sqrt_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -755,7 +755,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gaussian_identity_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_gaussian_identity_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -783,7 +783,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gaussian_log_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_gaussian_log_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -811,7 +811,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gaussian_inverse_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_gaussian_inverse_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -838,7 +838,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gamma_log_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_gamma_log_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -866,7 +866,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gamma_inverse_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_gamma_inverse_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -893,7 +893,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gamma_identity_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_gamma_identity_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -921,7 +921,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_binomial_probit_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_binomial_probit_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -949,7 +949,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_identity_transit
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_identity_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -961,7 +961,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_binomial_logit_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_binomial_logit_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -986,7 +986,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_result_z_stats(
/*+ state */ MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.__glm_result_type
AS 'MODULE_PATHNAME', 'glm_result_z_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1014,7 +1014,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_log_transition(
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_log_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1042,7 +1042,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_inverse_transiti
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_inverse_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1070,7 +1070,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_sqr_inverse_tran
MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_sqr_inverse_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1095,7 +1095,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_result_z_stats(
/*+ state */ MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.__glm_result_type
AS 'MODULE_PATHNAME', 'glm_result_z_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1104,7 +1104,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_result_t_stats(
/*+ state */ MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.__glm_result_type
AS 'MODULE_PATHNAME', 'glm_result_t_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1114,7 +1114,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_loglik_diff(
/*+ state2 */ MADLIB_SCHEMA.bytea8)
RETURNS double precision
AS 'MODULE_PATHNAME', 'glm_loglik_diff'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1212,7 +1212,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.glm_predict(
link TEXT
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME', 'glm_predict'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1236,7 +1236,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.glm_predict_binomial(
link TEXT
) RETURNS BOOLEAN
AS 'MODULE_PATHNAME', 'glm_predict_binomial'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1260,7 +1260,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.glm_predict_poisson(
link TEXT
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME', 'glm_predict_poisson'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/glm/multiresponseglm.sql_in b/src/ports/postgres/modules/glm/multiresponseglm.sql_in
index c810305..46de254 100644
--- a/src/ports/postgres/modules/glm/multiresponseglm.sql_in
+++ b/src/ports/postgres/modules/glm/multiresponseglm.sql_in
@@ -556,7 +556,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_merge_states(
state2 MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'multi_response_glm_merge_states'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -565,7 +565,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_final(
state MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'multi_response_glm_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -579,7 +579,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_logit_transition(
smallint)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'multi_response_glm_multinom_logit_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -605,7 +605,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_result(
/*+ state */ MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.__multinom_result_type
AS 'MODULE_PATHNAME', 'multi_response_glm_result_z_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -615,7 +615,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_loglik_diff(
/*+ state2 */ MADLIB_SCHEMA.bytea8)
RETURNS double precision
AS 'MODULE_PATHNAME', 'multi_response_glm_loglik_diff'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/glm/ordinal.sql_in b/src/ports/postgres/modules/glm/ordinal.sql_in
index 7cda294..366f69c 100644
--- a/src/ports/postgres/modules/glm/ordinal.sql_in
+++ b/src/ports/postgres/modules/glm/ordinal.sql_in
@@ -492,7 +492,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_merge_states(
state2 MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'ordinal_merge_states'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -501,7 +501,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_final(
state MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'ordinal_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -514,7 +514,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_logit_transition(
smallint)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'ordinal_logit_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -544,7 +544,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_probit_transition(
smallint)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME', 'ordinal_probit_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -570,7 +570,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_result(
/*+ state */ MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.__ordinal_result_type
AS 'MODULE_PATHNAME', 'ordinal_result'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -580,7 +580,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_loglik_diff(
/*+ state2 */ MADLIB_SCHEMA.bytea8)
RETURNS double precision
AS 'MODULE_PATHNAME', 'ordinal_loglik_diff'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/glm/test/multinom.sql_in b/src/ports/postgres/modules/glm/test/multinom.sql_in
index 936aa08..9ca1a1f 100644
--- a/src/ports/postgres/modules/glm/test/multinom.sql_in
+++ b/src/ports/postgres/modules/glm/test/multinom.sql_in
@@ -39,7 +39,8 @@ CREATE TABLE multinom_test (
feat1 INTEGER,
feat2 INTEGER,
cat INTEGER,
- g CHAR
+ g CHAR,
+ i serial
);
INSERT INTO multinom_test(feat1, feat2, cat, g) VALUES
@@ -274,7 +275,7 @@ WHERE category = '1';
-- for prediction
-- adding ID column for prediction
-ALTER TABLE multinom_test ADD COLUMN i SERIAL;
+-- ALTER TABLE multinom_test ADD COLUMN i SERIAL;
DROP TABLE IF EXISTS mglm_predict;
SELECT multinom_predict(
diff --git a/src/ports/postgres/modules/kmeans/kmeans.sql_in b/src/ports/postgres/modules/kmeans/kmeans.sql_in
index 9e36a92..afa9a4d 100644
--- a/src/ports/postgres/modules/kmeans/kmeans.sql_in
+++ b/src/ports/postgres/modules/kmeans/kmeans.sql_in
@@ -996,7 +996,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeans_args(
) RETURNS VOID
VOLATILE
CALLED ON NULL INPUT
-LANGUAGE c
+LANGUAGE c NOT FENCED
AS 'MODULE_PATHNAME', 'exec_sql_using'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1202,22 +1202,22 @@ BEGIN
-- Handle PG11 pg_proc table changes
IF (SELECT MADLIB_SCHEMA.is_pg_major_version_less_than(11) = TRUE) THEN
- IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = TRUE
+ IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = 't'
FROM pg_proc WHERE oid = proc_fn_dist) THEN
RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.';
END IF;
proc_agg_centroid := agg_centroid || '(DOUBLE PRECISION[])';
- IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR proisagg = FALSE
+ IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR proisagg != 't'
FROM pg_proc WHERE oid = proc_agg_centroid) THEN
RAISE EXCEPTION 'Kmeans error: Mean aggregate has wrong signature or is not an aggregate.';
END IF;
ELSE
- IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR prokind = 'a'
+ IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = 't'
FROM pg_proc WHERE oid = proc_fn_dist) THEN
RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.';
END IF;
proc_agg_centroid := agg_centroid || '(DOUBLE PRECISION[])';
- IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR prokind != 'a'
+ IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR proisagg != 't'
FROM pg_proc WHERE oid = proc_agg_centroid) THEN
RAISE EXCEPTION 'Kmeans error: Mean aggregate has wrong signature or is not an aggregate.';
END IF;
@@ -1255,8 +1255,8 @@ BEGIN
-- PostgreSQL 8.4:
-- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN
-- We therefore have to emulate.
+ DROP TABLE IF EXISTS pg_temp._madlib_kmeans_args;
PERFORM MADLIB_SCHEMA.internal_execute_using_kmeans_args($sql$
- DROP TABLE IF EXISTS pg_temp._madlib_kmeans_args;
CREATE TABLE pg_temp._madlib_kmeans_args AS
SELECT
$1 AS initial_centroids,
@@ -1354,7 +1354,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeanspp_seeding
) RETURNS VOID
VOLATILE
CALLED ON NULL INPUT
-LANGUAGE c
+LANGUAGE c NOT FENCED
AS 'MODULE_PATHNAME', 'exec_sql_using'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1414,7 +1414,7 @@ BEGIN
proc_fn_dist := fn_dist || '(DOUBLE PRECISION[], DOUBLE PRECISION[])';
IF (SELECT MADLIB_SCHEMA.is_pg_major_version_less_than(11) = TRUE) THEN
- IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = TRUE
+ IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR prokind = 'a'
FROM pg_proc WHERE oid = proc_fn_dist) THEN
RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.';
END IF;
@@ -1463,8 +1463,8 @@ BEGIN
-- PostgreSQL 8.4:
-- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN
-- We therefore have to emulate.
+ DROP TABLE IF EXISTS pg_temp._madlib_kmeanspp_args;
PERFORM MADLIB_SCHEMA.internal_execute_using_kmeanspp_seeding_args($sql$
- DROP TABLE IF EXISTS pg_temp._madlib_kmeanspp_args;
CREATE TEMPORARY TABLE _madlib_kmeanspp_args AS
SELECT
$1 AS k,
@@ -1685,7 +1685,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeans_random_se
) RETURNS VOID
VOLATILE
CALLED ON NULL INPUT
-LANGUAGE c
+LANGUAGE c NOT FENCED
AS 'MODULE_PATHNAME', 'exec_sql_using'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1745,8 +1745,8 @@ BEGIN
-- PostgreSQL 8.4:
-- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN
-- We therefore have to emulate.
+ DROP TABLE IF EXISTS pg_temp._madlib_kmeans_random_args;
PERFORM MADLIB_SCHEMA.internal_execute_using_kmeans_random_seeding_args($sql$
- DROP TABLE IF EXISTS pg_temp._madlib_kmeans_random_args;
CREATE TEMPORARY TABLE _madlib_kmeans_random_args AS
SELECT $1 AS k, $2 AS initial_centroids;
$sql$,
@@ -1924,7 +1924,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeans_args(
) RETURNS MADLIB_SCHEMA.kmeans_result
VOLATILE
CALLED ON NULL INPUT
-LANGUAGE c
+LANGUAGE c NOT FENCED
AS 'MODULE_PATHNAME', 'exec_sql_using'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -2060,7 +2060,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_silhouette_args(
) RETURNS DOUBLE PRECISION
STABLE
CALLED ON NULL INPUT
-LANGUAGE c
+LANGUAGE c NOT FENCED
AS 'MODULE_PATHNAME', 'exec_sql_using'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -2117,7 +2117,7 @@ BEGIN
proc_fn_dist := fn_dist
|| '(DOUBLE PRECISION[], DOUBLE PRECISION[])';
IF (SELECT MADLIB_SCHEMA.is_pg_major_version_less_than(11) = TRUE) THEN
- IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = TRUE
+ IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR prokind = 'a'
FROM pg_proc WHERE oid = proc_fn_dist) THEN
RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.';
END IF;
diff --git a/src/ports/postgres/modules/knn/knn.py_in b/src/ports/postgres/modules/knn/knn.py_in
index eb2150e..1f5f055 100644
--- a/src/ports/postgres/modules/knn/knn.py_in
+++ b/src/ports/postgres/modules/knn/knn.py_in
@@ -122,7 +122,7 @@ def knn_validate_src(schema_madlib, point_source, point_column_name, point_id,
if fn_dist:
fn_dist = fn_dist.lower().strip()
- profunc = ("proisagg = TRUE"
+ profunc = ("prokind = 'a'"
if is_pg_major_version_less_than(schema_madlib, 11)
else "prokind = 'a'")
@@ -180,12 +180,13 @@ def build_kd_tree(schema_madlib, source_table, output_table, point_column_name,
for curr_leaf in range(pow(2,curr_level)):
clause = clauses[clause_counter]
cutoff_sql = """
- SELECT percentile_disc(0.5)
- WITHIN GROUP (
+ SELECT {schema_madlib}.percentile_disc(array_agg(cutoff), 0.5) AS cutoff
+ FROM (
+ SELECT {point_column_name}[{curr_feature}] AS cutoff
+ FROM {source_table}
+ WHERE {clause} and cutoff is not null
ORDER BY ({point_column_name})[{curr_feature}]
- ) AS cutoff
- FROM {source_table}
- WHERE {clause}
+ )
""".format(**locals())
cutoff = plpy.execute(cutoff_sql)[0]['cutoff']
diff --git a/src/ports/postgres/modules/knn/test/knn.sql_in b/src/ports/postgres/modules/knn/test/knn.sql_in
index e4cd578..63a9883 100644
--- a/src/ports/postgres/modules/knn/test/knn.sql_in
+++ b/src/ports/postgres/modules/knn/test/knn.sql_in
@@ -143,18 +143,18 @@ SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0.040872859187601
DROP TABLE if exists madlib_knn_result_classification;
-SELECT knn('knn_train_data','data[1:1]||data[2:2]','id','label','knn_test_data','data[1:1]||data[2:2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True);
+SELECT knn('knn_train_data','data[1,1]||data[2,2]','id','label','knn_test_data','data[1,1]||data[2,2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True);
SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0,1,0,0}', 'Wrong output in classification') FROM madlib_knn_result_classification;
DROP TABLE if exists madlib_knn_result_regression;
-SELECT knn('knn_train_data_reg','data[1:1]||data[2:2]','id','label','knn_test_data','data[1:1]||data[2:2]','id','madlib_knn_result_regression',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True);
+SELECT knn('knn_train_data_reg','data[1,1]||data[2,2]','id','label','knn_test_data','data[1,1]||data[2,2]','id','madlib_knn_result_regression',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True);
SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0.0408728591876018,1,0,0}', 'Wrong output in regression') FROM madlib_knn_result_regression;
DROP TABLE if exists madlib_knn_result_classification;
-SELECT knn('knn_train_data_expr','ARRAY[data1,data2]','id','label','knn_test_data','data[1:1]||data[2:2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True);
+SELECT knn('knn_train_data_expr','ARRAY[data1,data2]','id','label','knn_test_data','data[1,1]||data[2,2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True);
SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0,1,0,0}', 'Wrong output in classification') FROM madlib_knn_result_classification;
diff --git a/src/ports/postgres/modules/lda/lda.py_in b/src/ports/postgres/modules/lda/lda.py_in
index 29c9d85..c34ced5 100644
--- a/src/ports/postgres/modules/lda/lda.py_in
+++ b/src/ports/postgres/modules/lda/lda.py_in
@@ -229,7 +229,7 @@ class LDATrainer:
{schema_madlib}.__lda_count_topic_agg(
words,
counts,
- doc_topic[{topic_num} + 1:array_upper(doc_topic, 1)],
+ doc_topic[{topic_num} + 1,array_upper(doc_topic, 1)],
{voc_size},
{topic_num}
) AS model
@@ -253,7 +253,7 @@ class LDATrainer:
{schema_madlib}.__lda_count_topic_agg(
words,
counts,
- doc_topic[{topic_num} + 1:array_upper(doc_topic, 1)],
+ doc_topic[{topic_num} + 1,array_upper(doc_topic, 1)],
{voc_size},
{topic_num}
) AS model
@@ -319,8 +319,8 @@ class LDATrainer:
plpy.execute("""
INSERT INTO {output_data_table}
SELECT
- docid, wordcount, words, counts, doc_topic[1:{topic_num}] topic_count,
- doc_topic[{topic_num} + 1:array_upper(doc_topic,1)] topic_assignment
+ docid, wordcount, words, counts, doc_topic[1,{topic_num}] topic_count,
+ doc_topic[{topic_num} + 1,array_upper(doc_topic,1)] topic_assignment
FROM
{work_table_final}
""".format(output_data_table=self.output_data_table,
@@ -448,8 +448,8 @@ class LDAPredictor:
plpy.execute("""
INSERT INTO {doc_topic}
SELECT
- docid, wordcount, words, counts, doc_topic[1:{topic_num}] topic_count,
- doc_topic[{topic_num} + 1:array_upper(doc_topic,1)] topic_assignment
+ docid, wordcount, words, counts, doc_topic[1,{topic_num}] topic_count,
+ doc_topic[{topic_num} + 1,array_upper(doc_topic,1)] topic_assignment
FROM {work_table_out}
""".format(doc_topic=self.doc_topic,
topic_num=self.topic_num,
@@ -1116,7 +1116,7 @@ def _validate_data_table(data_table, voc_size):
rv = plpy.execute("""
SELECT
- count(wordid) size,
+ count(wordid) AS size,
min(wordid) min_wordid,
max(wordid) max_wordid
FROM
diff --git a/src/ports/postgres/modules/lda/lda.sql_in b/src/ports/postgres/modules/lda/lda.sql_in
index 814b0ae..a0910ca 100644
--- a/src/ports/postgres/modules/lda/lda.sql_in
+++ b/src/ports/postgres/modules/lda/lda.sql_in
@@ -1264,7 +1264,7 @@ MADLIB_SCHEMA.__lda_random_assign
)
RETURNS INT4[]
AS 'MODULE_PATHNAME', 'lda_random_assign'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1302,7 +1302,7 @@ MADLIB_SCHEMA.__lda_gibbs_sample
)
RETURNS INT4[]
AS 'MODULE_PATHNAME', 'lda_gibbs_sample'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1330,7 +1330,7 @@ MADLIB_SCHEMA.__lda_count_topic_sfunc
)
RETURNS INT8[]
AS 'MODULE_PATHNAME', 'lda_count_topic_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1348,7 +1348,7 @@ MADLIB_SCHEMA.__lda_count_topic_prefunc
)
RETURNS INT8[]
AS 'MODULE_PATHNAME', 'lda_count_topic_prefunc'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1436,7 +1436,7 @@ MADLIB_SCHEMA.__lda_perplexity_sfunc
)
RETURNS INT8[]
AS 'MODULE_PATHNAME', 'lda_perplexity_sfunc'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1453,7 +1453,7 @@ MADLIB_SCHEMA.__lda_perplexity_prefunc
)
RETURNS INT8[]
AS 'MODULE_PATHNAME', 'lda_perplexity_prefunc'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1468,7 +1468,7 @@ MADLIB_SCHEMA.__lda_perplexity_ffunc
)
RETURNS FLOAT8
AS 'MODULE_PATHNAME', 'lda_perplexity_ffunc'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
@@ -1526,7 +1526,7 @@ MADLIB_SCHEMA.__lda_check_count_ceiling
)
RETURNS INT4[]
AS 'MODULE_PATHNAME', 'lda_check_count_ceiling'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1543,7 +1543,7 @@ MADLIB_SCHEMA.__lda_util_unnest
)
RETURNS SETOF INT4[]
AS 'MODULE_PATHNAME', 'lda_unnest'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -1555,7 +1555,7 @@ MADLIB_SCHEMA.__lda_util_unnest_transpose
)
RETURNS SETOF INT4[]
AS 'MODULE_PATHNAME', 'lda_unnest_transpose'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1570,7 +1570,7 @@ MADLIB_SCHEMA.__lda_util_transpose
)
RETURNS INT8[][]
AS 'MODULE_PATHNAME', 'lda_transpose'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1587,7 +1587,7 @@ MADLIB_SCHEMA.__lda_util_norm_with_smoothing
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'l1_norm_with_smoothing'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1694,6 +1694,6 @@ MADLIB_SCHEMA.lda_parse_model
)
RETURNS MADLIB_SCHEMA._pivotalr_lda_model
AS 'MODULE_PATHNAME'
-LANGUAGE c STRICT
+LANGUAGE c STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/lda/test/lda.sql_in b/src/ports/postgres/modules/lda/test/lda.sql_in
index 96d5f5f..0dc3b9a 100644
--- a/src/ports/postgres/modules/lda/test/lda.sql_in
+++ b/src/ports/postgres/modules/lda/test/lda.sql_in
@@ -276,7 +276,7 @@ CREATE OR REPLACE FUNCTION validate_lda_output() RETURNS integer AS $$
-- Loop through the output of lda_get_word_topic_count() and compare the results with the helper table word_topic_mapping
FOR word_topic_count_row IN select * from word_topic_count ORDER BY wordid LOOP
word_topic_count_topic_count := ARRAY[word_topic_count_row.topic_count];
- word_topic_mapping_topic_count := word_topic_mapping_array[word_topic_count_row.wordid+1:word_topic_count_row.wordid+1]; -- Here arrayX[i:i] means the ith 1d array of a 2d array
+ word_topic_mapping_topic_count := word_topic_mapping_array[word_topic_count_row.wordid+1,word_topic_count_row.wordid+1]; -- Here arrayX[i:i] means the ith 1d array of a 2d array
IF (word_topic_mapping_topic_count != word_topic_count_topic_count) THEN
RAISE EXCEPTION 'Topic assignment for wordid % does not match. word_topic_mapping: % word_topic_count: %',word_topic_count_row.wordid, word_topic_mapping_topic_count, word_topic_count_topic_count;
diff --git a/src/ports/postgres/modules/linalg/linalg.sql_in b/src/ports/postgres/modules/linalg/linalg.sql_in
index 3c74451..7570ae1 100644
--- a/src/ports/postgres/modules/linalg/linalg.sql_in
+++ b/src/ports/postgres/modules/linalg/linalg.sql_in
@@ -227,7 +227,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.norm1(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -242,7 +242,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.norm2(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -259,7 +259,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_inf_norm(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -278,7 +278,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_pnorm(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -295,7 +295,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_norm1(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -312,7 +312,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_norm2(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -331,7 +331,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cosine_similarity(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -350,7 +350,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.squared_dist_norm2(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -368,7 +368,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_angle(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -387,7 +387,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_tanimoto(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -405,7 +405,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_jaccard(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/*
@@ -424,7 +424,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._closest_column(
dist_dn TEXT
) RETURNS MADLIB_SCHEMA.closest_column_result AS
'MODULE_PATHNAME', 'closest_column'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
@@ -508,7 +508,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._closest_columns(
dist_dn TEXT
) RETURNS MADLIB_SCHEMA.closest_columns_result AS
'MODULE_PATHNAME', 'closest_columns'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.closest_columns(
@@ -541,7 +541,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_vector_transition(
x DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
-IMMUTABLE
+IMMUTABLE NOT FENCED
CALLED ON NULL INPUT
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -552,7 +552,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_vector_merge(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -561,7 +561,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_vector_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -590,7 +590,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalized_avg_vector_transition(
x DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
-IMMUTABLE
+IMMUTABLE NOT FENCED
CALLED ON NULL INPUT
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -600,7 +600,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalized_avg_vector_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -632,7 +632,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.matrix_agg_transition(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -641,7 +641,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.matrix_agg_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -676,7 +676,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.matrix_column(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -690,7 +690,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.deconstruct_2d_array(
in_array double precision[]
) RETURNS SETOF record
AS 'MODULE_PATHNAME', 'deconstruct_2d_array'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
------------------------------
@@ -703,7 +703,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__deconstruct_lower_triangle(
in_array double precision[]
) RETURNS SETOF record
AS 'MODULE_PATHNAME', 'deconstruct_lower_triangle'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
------------------- Created for In-Memory Iteration Controller -----------
@@ -716,7 +716,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_to_1d(
in_array DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'array_to_1d'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -729,7 +729,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_to_2d(
in_array DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'array_to_2d'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -744,7 +744,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.index_2d_array(
)
RETURNS double precision[]
AS 'MODULE_PATHNAME', 'get_row_from_2d_array'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -759,7 +759,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.get_row(
)
RETURNS double precision[]
AS 'MODULE_PATHNAME', 'get_row_from_2d_array'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -774,7 +774,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.get_col(
)
RETURNS double precision[]
AS 'MODULE_PATHNAME', 'get_col_from_2d_array'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
m4_changequote(<!`!>, <!'!>)
diff --git a/src/ports/postgres/modules/linalg/matrix_ops.py_in b/src/ports/postgres/modules/linalg/matrix_ops.py_in
index bdbb502..b03d26d 100644
--- a/src/ports/postgres/modules/linalg/matrix_ops.py_in
+++ b/src/ports/postgres/modules/linalg/matrix_ops.py_in
@@ -2570,10 +2570,12 @@ def matrix_cholesky(schema_madlib, matrix_in, in_args,
plpy.execute('DROP TABLE IF EXISTS %s' % matrix_p_temp)
matrix_l_temp = "pg_temp." + unique_string() if is_output_sparse else matrix_l
+ dim_start = dim[0] + 1
+ dim_end = dim[0] * 2
plpy.execute("""
CREATE TABLE {matrix_l_temp} AS
SELECT row_id AS {out_args[row]},
- row_vec[{dim[0]} + 1 : {dim[0]} * 2] AS {out_args[val]}
+ row_vec[{dim_start}:{dim_end}] AS {out_args[val]}
FROM {matrix_temp}
""".format(**locals()))
if is_output_sparse:
@@ -2581,10 +2583,12 @@ def matrix_cholesky(schema_madlib, matrix_in, in_args,
plpy.execute('DROP TABLE IF EXISTS %s' % matrix_l_temp)
matrix_d_temp = "pg_temp." + unique_string() if is_output_sparse else matrix_d
+ dim_start = dim[0] * 2 + 1
+ dim_end = dim[0] * 3
plpy.execute("""
CREATE TABLE {matrix_d_temp} AS
SELECT row_id AS {out_args[row]},
- row_vec[{dim[0]} * 2 + 1: {dim[0]} * 3] AS {out_args[val]}
+ row_vec[{dim_start}:{dim_end}] AS {out_args[val]}
FROM {matrix_temp}
""".format(**locals()))
if is_output_sparse:
@@ -2673,10 +2677,12 @@ def matrix_qr(schema_madlib, matrix_in, in_args, matrix_out_prefix, out_args):
plpy.execute('DROP TABLE IF EXISTS %s' % matrix_q_temp)
matrix_r_temp = "pg_temp." + unique_string() if is_output_sparse else matrix_r
+ dim_start = dim[0] + 1
+ dim_end = dim[0] + dim[1]
plpy.execute("""
CREATE TABLE {matrix_r_temp} AS
SELECT row_id as {out_args[row]},
- row_vec[{dim[0]} + 1 : {dim[0]} + {dim[1]}] AS {out_args[val]}
+ row_vec[{dim_start}:{dim_end}] AS {out_args[val]}
FROM {matrix_temp}
""".format(**locals()))
if is_output_sparse:
@@ -2829,22 +2835,28 @@ def matrix_lu(schema_madlib, matrix_in, in_args,
row_vec[1:{dim[0]}] AS {out_args[val]}
FROM {matrix_temp} WHERE row_id <= {dim[0]}
""".format(**locals()))
+ dim_start = 2 * dim[0] + dim[1] + 1
+ dim_end = 2 * dim[0] + 2 * dim[1]
plpy.execute("""
CREATE TABLE {matrix_temp_names[q]} AS
SELECT row_id AS {out_args[row]},
- row_vec[2 * {dim[0]} + {dim[1]} + 1:2 * {dim[0]} + 2 * {dim[1]}] AS {out_args[val]}
+ row_vec[{dim_start} : {dim_end}] AS {out_args[val]}
FROM {matrix_temp} WHERE row_id <= {dim[1]}
""".format(**locals()))
+ dim_start = dim[0] + 1
+ dim_end = 2 * dim[0]
plpy.execute("""
CREATE TABLE {matrix_temp_names[l]} AS
SELECT row_id AS {out_args[row]},
- row_vec[{dim[0]} + 1 : 2 * {dim[0]}] AS {out_args[val]}
+ row_vec[{dim_start} : {dim_end}] AS {out_args[val]}
FROM {matrix_temp} WHERE row_id <= {dim[0]}
""".format(**locals()))
+ dim_start = 2 * dim[0] + 1
+ dim_end = 2 * dim[0] + dim[1]
plpy.execute("""
CREATE TABLE {matrix_temp_names[u]} AS
SELECT row_id AS {out_args[row]},
- row_vec[2 * {dim[0]} + 1 : 2 * {dim[0]} + {dim[1]}] AS {out_args[val]}
+ row_vec[{dim_start} : {dim_end}] AS {out_args[val]}
FROM {matrix_temp} WHERE row_id <= {dim[0]}
""".format(**locals()))
diff --git a/src/ports/postgres/modules/linalg/matrix_ops.sql_in b/src/ports/postgres/modules/linalg/matrix_ops.sql_in
index 6e33f46..06eb944 100644
--- a/src/ports/postgres/modules/linalg/matrix_ops.sql_in
+++ b/src/ports/postgres/modules/linalg/matrix_ops.sql_in
@@ -3000,7 +3000,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_vec_mult_in_mem(
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_vec_mult_in_mem_2d'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/*
@@ -3015,7 +3015,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_vec_mult_in_mem(
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_vec_mult_in_mem_1d'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -3031,7 +3031,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__row_fold(
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'row_fold'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -3045,7 +3045,7 @@ MADLIB_SCHEMA.__bernoulli_vector
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'bernoulli_vector'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -3068,7 +3068,7 @@ MADLIB_SCHEMA.__uniform_vector
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'uniform_vector'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -3093,7 +3093,7 @@ MADLIB_SCHEMA.__normal_vector
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'normal_vector'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -3116,7 +3116,7 @@ MADLIB_SCHEMA.__rand_vector
)
RETURNS INTEGER[]
AS 'MODULE_PATHNAME', 'rand_vector'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -3128,7 +3128,7 @@ MADLIB_SCHEMA.__rand_block
)
RETURNS INTEGER[]
AS 'MODULE_PATHNAME', 'rand_block'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -3149,7 +3149,7 @@ MADLIB_SCHEMA.__matrix_row_split
)
RETURNS SETOF FLOAT8[]
AS 'MODULE_PATHNAME', 'row_split'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -3162,7 +3162,7 @@ MADLIB_SCHEMA.__matrix_densify_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_densify_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS
@@ -3222,7 +3222,7 @@ MADLIB_SCHEMA.__matrix_blockize_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_blockize_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS
@@ -3255,7 +3255,7 @@ MADLIB_SCHEMA.__matrix_unblockize_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_unblockize_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS
@@ -3287,7 +3287,7 @@ MADLIB_SCHEMA.matrix_mem_mult
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_mem_mult'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -3310,7 +3310,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
-- )
-- RETURNS FLOAT8[]
-- AS 'MODULE_PATHNAME', 'matrix_vec_mem_mult'
--- $$ LANGUAGE C;
+-- $$ LANGUAGE C NOT FENCED;
CREATE OR REPLACE FUNCTION
@@ -3320,7 +3320,7 @@ MADLIB_SCHEMA.matrix_mem_trans
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_mem_trans'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -3332,7 +3332,7 @@ MADLIB_SCHEMA.__matrix_mem_sum_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'matrix_mem_sum_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS
@@ -3427,7 +3427,7 @@ MADLIB_SCHEMA.__matrix_unnest_block
)
RETURNS SETOF FLOAT8[]
AS 'MODULE_PATHNAME', 'unnest_block'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------------------------------------------------------------------
@@ -3439,7 +3439,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_compose_dense_transition(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_compose_dense_transition'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3453,7 +3453,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_compose_sparse_transition(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_compose_sparse_transition'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3463,7 +3463,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_compose_merge(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_compose_merge'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3472,7 +3472,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_inv_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_inv'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3481,7 +3481,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_pinv_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_pinv'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3490,7 +3490,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_eigen_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_eigen'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3499,7 +3499,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_cholesky_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_cholesky'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3508,7 +3508,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_qr_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_qr'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3517,7 +3517,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_rank_final(
) RETURNS INT8
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_rank'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3526,7 +3526,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_lu_final(
) RETURNS DOUBLE PRECISION[]
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_lu'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
@@ -3535,7 +3535,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_nuclear_norm_final(
) RETURNS DOUBLE PRECISION
LANGUAGE c
IMMUTABLE
-STRICT
+STRICT NOT FENCED
AS 'MODULE_PATHNAME', 'matrix_nuclear_norm'
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
diff --git a/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in b/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in
index ff569d8..0f069ee 100644
--- a/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in
+++ b/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in
@@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_transition(
right_arr DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', '__pivotalr_crossprod_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_merge(
@@ -14,7 +14,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_merge(
right_state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', '__pivotalr_crossprod_merge'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.crossprod(DOUBLE PRECISION[], DOUBLE PRECISION[]);
@@ -36,7 +36,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_sym_transition(
arr DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', '__pivotalr_crossprod_sym_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.crossprod_sym(DOUBLE PRECISION[]);
@@ -57,14 +57,14 @@ CREATE AGGREGATE MADLIB_SCHEMA.crossprod_sym(
-- arr DOUBLE PRECISION[]
-- ) RETURNS DOUBLE PRECISION[] AS
-- 'MODULE_PATHNAME', '__pivotalr_array_mean_transition'
--- LANGUAGE C IMMUTABLE;
+-- LANGUAGE C IMMUTABLE NOT FENCED;
-- CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_array_sum_merge(
-- left_state DOUBLE PRECISION[],
-- right_state DOUBLE PRECISION[]
-- ) RETURNS DOUBLE PRECISION[] AS
-- 'MODULE_PATHNAME', '__pivotalr_array_mean_merge'
--- LANGUAGE C IMMUTABLE;
+-- LANGUAGE C IMMUTABLE NOT FENCED;
-- CREATE AGGREGATE MADLIB_SCHEMA.pivotalr_array_sum(
-- /* vector */ DOUBLE PRECISION[]
diff --git a/src/ports/postgres/modules/linalg/svd.sql_in b/src/ports/postgres/modules/linalg/svd.sql_in
index 50313c5..a7115ac 100644
--- a/src/ports/postgres/modules/linalg/svd.sql_in
+++ b/src/ports/postgres/modules/linalg/svd.sql_in
@@ -677,7 +677,7 @@ MADLIB_SCHEMA.__svd_unit_vector
-- RETURNS MADLIB_SCHEMA.__svd_unit_vector_result
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'svd_unit_vector'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP TYPE IF EXISTS MADLIB_SCHEMA.__svd_lanczos_result CASCADE;
@@ -705,7 +705,7 @@ MADLIB_SCHEMA.__svd_lanczos_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_lanczos_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------------------------------------------
@@ -723,7 +723,7 @@ MADLIB_SCHEMA.__svd_block_lanczos_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_block_lanczos_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------------------------------------------
@@ -741,7 +741,7 @@ MADLIB_SCHEMA.__svd_sparse_lanczos_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_sparse_lanczos_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -752,7 +752,7 @@ MADLIB_SCHEMA.__svd_lanczos_prefunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_lanczos_prefunc'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------------------------------------------
@@ -860,7 +860,7 @@ MADLIB_SCHEMA.__svd_lanczos_pvec
)
RETURNS MADLIB_SCHEMA.__svd_lanczos_result
AS 'MODULE_PATHNAME', 'svd_lanczos_pvec'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -872,7 +872,7 @@ MADLIB_SCHEMA.__svd_lanczos_qvec
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_lanczos_qvec'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------------------------------------------
@@ -887,7 +887,7 @@ MADLIB_SCHEMA.__svd_gram_schmidt_orthogonalize_sfunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_gram_schmidt_orthogonalize_sfunc'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION
@@ -898,7 +898,7 @@ MADLIB_SCHEMA.__svd_gram_schmidt_orthogonalize_prefunc
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_gram_schmidt_orthogonalize_prefunc'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-- This function will also do the normalization
@@ -909,7 +909,7 @@ MADLIB_SCHEMA.__svd_gram_schmidt_orthogonalize_ffunc
)
RETURNS MADLIB_SCHEMA.__svd_lanczos_result
AS 'MODULE_PATHNAME', 'svd_gram_schmidt_orthogonalize_ffunc'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS
@@ -1003,7 +1003,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__svd_decompose_bidiag(
)
RETURNS MADLIB_SCHEMA.__svd_bidiagonal_matrix_result
AS 'MODULE_PATHNAME', 'svd_decompose_bidiag'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-----------------------------------------------------------------------
@@ -1022,7 +1022,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__svd_vec_mult_matrix(
)
RETURNS FLOAT8[]
AS 'MODULE_PATHNAME', 'svd_vec_mult_matrix'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP TYPE IF EXISTS MADLIB_SCHEMA.__svd_vec_mat_mult_result CASCADE;
@@ -1042,7 +1042,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__svd_vec_trans_mult_matrix_internal(
)
RETURNS SETOF MADLIB_SCHEMA.__svd_vec_mat_mult_result
AS 'MODULE_PATHNAME', 'svd_vec_trans_mult_matrix'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--Multiplication of P/Q vectors with Left/Right Singular Matrix of B
diff --git a/src/ports/postgres/modules/linalg/test/linalg.sql_in b/src/ports/postgres/modules/linalg/test/linalg.sql_in
index 4777d8d..b20bd44 100644
--- a/src/ports/postgres/modules/linalg/test/linalg.sql_in
+++ b/src/ports/postgres/modules/linalg/test/linalg.sql_in
@@ -127,7 +127,7 @@ SELECT assert(
'Incorrect closest column.'
) FROM (
SELECT
- squared_dist_norm2(matrix[column_id + lb:column_id + lb], x) AS dist1,
+ squared_dist_norm2(matrix[column_id + lb,column_id + lb], x) AS dist1,
distance AS dist2,
least AS dist3,
*
diff --git a/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in b/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in
index 8a7eb95..234373f 100644
--- a/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in
+++ b/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in
@@ -216,3 +216,22 @@ FROM (
SELECT array_to_1d(__matrix_sparse_inverse(3, 3, row_id - 1, col_id - 1, val)) as result
FROM random_sparse_matrix
) q;
+
+CREATE TABLE "mat_A" (
+ row_id integer,
+ row_vec integer[]
+);
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (1, '{9,6,5,8,5,6,6,3,10,8}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (2, '{8,2,2,6,6,10,2,1,9,9}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (3, '{3,9,9,9,8,6,3,9,5,6}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (4, '{6,4,2,2,2,7,8,8,0,7}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (5, '{6,8,9,9,4,6,9,5,7,7}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (6, '{4,10,7,3,9,5,9,2,3,4}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (7, '{8,10,7,10,1,9,7,9,8,7}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (8, '{7,4,5,6,2,8,1,1,4,8}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (9, '{8,8,8,5,2,6,9,1,8,3}');
+INSERT INTO "mat_A" (row_id, row_vec) VALUES (10, '{4,6,3,2,6,4,1,2,3,8}');
+
+SELECT madlib.matrix_cholesky('"mat_A"', 'row=row_id, val=row_vec', 'matrix_out_prefix1');
+SELECT madlib.matrix_qr('"mat_A"', 'row=row_id, val=row_vec', 'matrix_out_prefix2');
+SELECT madlib.matrix_lu('"mat_A"', 'row=row_id, val=row_vec', 'matrix_out_prefix3');
diff --git a/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in b/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in
index 15886f7..b1f2777 100644
--- a/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in
+++ b/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in
@@ -249,7 +249,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_transition(
x DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_merge_states(
@@ -257,14 +257,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_merge_states(
state2 MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_final(
state MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.residual_norm_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -315,7 +315,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_transition(
algorithm INTEGER)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_merge_states(
@@ -323,14 +323,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_merge_states
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.dense_linear_solver_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
diff --git a/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in b/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in
index a61a48e..a9ef39a 100644
--- a/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in
+++ b/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in
@@ -362,7 +362,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_tr
termToler DOUBLE PRECISION)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_merge_states(
@@ -370,14 +370,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_me
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.sparse_linear_solver_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -456,7 +456,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_transition(
algorithm INTEGER)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_merge_states(
@@ -464,7 +464,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_merge_state
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -472,7 +472,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.sparse_linear_solver_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/pca/pca.py_in b/src/ports/postgres/modules/pca/pca.py_in
index cab8e7f..3882f24 100644
--- a/src/ports/postgres/modules/pca/pca.py_in
+++ b/src/ports/postgres/modules/pca/pca.py_in
@@ -166,9 +166,9 @@ def pca_wrap(schema_madlib, source_table, pc_table, row_id,
# declare variables whose values will be different for each group, if
# grouping_cols is specified
- grouping_where_clause = ''
- sparse_where_condition = ''
- select_grouping_cols = ''
+ grouping_where_clause = ' '
+ sparse_where_condition = ' '
+ select_grouping_cols = ' '
temp_table_columns = ''
result_summary_table_temp = ''
# For Dense matrix format only:
diff --git a/src/ports/postgres/modules/pca/pca_project.py_in b/src/ports/postgres/modules/pca/pca_project.py_in
index d205044..64e97fd 100644
--- a/src/ports/postgres/modules/pca/pca_project.py_in
+++ b/src/ports/postgres/modules/pca/pca_project.py_in
@@ -416,10 +416,10 @@ def pca_project_wrap(schema_madlib, source_table, pc_table, out_table,
# declare variables whose values will be different for each group, if
# grouping_cols is specified
- grouping_where_clause = ''
- sparse_where_condition = ''
- select_grouping_cols = ''
- grouping_cols_values = ''
+ grouping_where_clause = ' '
+ sparse_where_condition = ' '
+ select_grouping_cols = ' '
+ grouping_cols_values = ' '
result_summary_table_temp = ''
other_columns_in_pc_table = [col for col in get_cols(pc_table) if col not in grouping_cols_list]
temp_pc_table_columns = ', '.join(other_columns_in_pc_table)
diff --git a/src/ports/postgres/modules/prob/prob.sql_in b/src/ports/postgres/modules/prob/prob.sql_in
index 5322c29..88dfe63 100644
--- a/src/ports/postgres/modules/prob/prob.sql_in
+++ b/src/ports/postgres/modules/prob/prob.sql_in
@@ -144,7 +144,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -162,9 +162,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_pmf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_pmf(
+ x DOUBLE PRECISION,
+ sp DOUBLE PRECISION
+) RETURNS VOID AS $$
+DECLARE
+BEGIN
+ RAISE EXCEPTION 'The first element must be INT';
+END
+$$ LANGUAGE plpgsql IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+
/**
* @brief Bernoulli quantile function
*
@@ -178,7 +189,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -198,7 +209,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.beta_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -218,7 +229,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.beta_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -238,7 +249,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.beta_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -259,7 +270,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.binomial_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -279,7 +290,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.binomial_pmf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -301,7 +312,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.binomial_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -322,7 +333,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cauchy_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -342,7 +353,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cauchy_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -362,7 +373,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cauchy_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -380,7 +391,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi_squared_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -398,7 +409,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi_squared_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -416,7 +427,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi_squared_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -434,7 +445,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.exponential_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -452,7 +463,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.exponential_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -470,7 +481,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.exponential_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -491,7 +502,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.extreme_value_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -511,7 +522,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.extreme_value_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -531,7 +542,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.extreme_value_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -551,7 +562,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fisher_f_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -571,7 +582,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fisher_f_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -591,7 +602,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fisher_f_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -612,7 +623,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -632,7 +643,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -652,7 +663,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -669,7 +680,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -687,9 +698,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_pmf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_pmf(
+ x DOUBLE PRECISION,
+ sp DOUBLE PRECISION
+) RETURNS VOID AS $$
+DECLARE
+BEGIN
+ RAISE EXCEPTION 'The first element must be INT';
+END
+$$ LANGUAGE plpgsql IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+
/**
* @brief Geometric quantile function
*
@@ -707,7 +729,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -735,7 +757,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hypergeometric_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -760,7 +782,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hypergeometric_pmf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -786,7 +808,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hypergeometric_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -807,7 +829,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.inverse_gamma_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -827,7 +849,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.inverse_gamma_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -847,7 +869,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.inverse_gamma_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -865,7 +887,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.kolmogorov_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -885,7 +907,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.laplace_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -905,7 +927,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.laplace_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -925,7 +947,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.laplace_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -945,7 +967,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logistic_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -965,7 +987,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logistic_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -985,7 +1007,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logistic_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1006,7 +1028,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lognormal_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1026,7 +1048,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lognormal_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1046,7 +1068,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lognormal_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1067,7 +1089,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.negative_binomial_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1088,7 +1110,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.negative_binomial_pmf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1111,7 +1133,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.negative_binomial_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1134,7 +1156,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_beta_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1157,7 +1179,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_beta_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1180,7 +1202,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_beta_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1201,7 +1223,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_chi_squared_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1221,7 +1243,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_chi_squared_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1241,7 +1263,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_chi_squared_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1264,7 +1286,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_f_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1286,7 +1308,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_f_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1308,7 +1330,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_f_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1329,7 +1351,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_t_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1349,7 +1371,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_t_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1369,7 +1391,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_t_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1389,7 +1411,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_cdf(
@@ -1428,7 +1450,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_pdf(
@@ -1469,7 +1491,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_quantile(
@@ -1511,7 +1533,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pareto_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1531,7 +1553,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pareto_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1551,7 +1573,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pareto_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1569,7 +1591,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1586,9 +1608,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_pmf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_pmf(
+ x DOUBLE PRECISION,
+ sp DOUBLE PRECISION
+) RETURNS VOID AS $$
+DECLARE
+BEGIN
+ RAISE EXCEPTION 'The first element must be INT';
+END
+$$ LANGUAGE plpgsql IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+
/**
* @brief Poisson quantile function
*
@@ -1605,7 +1638,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1623,7 +1656,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rayleigh_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1640,7 +1673,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rayleigh_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1657,7 +1690,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rayleigh_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1675,7 +1708,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.students_t_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1693,7 +1726,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.students_t_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1711,7 +1744,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.students_t_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1733,7 +1766,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.triangular_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1754,7 +1787,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.triangular_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1775,7 +1808,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.triangular_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1795,7 +1828,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.uniform_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1814,7 +1847,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.uniform_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1833,7 +1866,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.uniform_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1854,7 +1887,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weibull_cdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1874,7 +1907,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weibull_pdf(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1894,5 +1927,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weibull_quantile(
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index 69a54f3..2cffe99 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -398,7 +398,7 @@ def _build_tree(schema_madlib, is_classification, split_criterion,
FROM {1}
""".format(grouping_array_str, cp_table)
grp_cp_values = plpy.execute(sql)
- grp_key_to_cp = dict((row['grp_key'], row['cp_val']) for row in grp_cp_values)
+ grp_key_to_cp = dict((""if row['grp_key'] is None else row['grp_key'], row['cp_val']) for row in grp_cp_values)
with MinWarning(msg_level):
plpy.notice("Building tree for cross validation")
@@ -2144,7 +2144,7 @@ def _xvalidate(schema_madlib, tree_states, training_table_name, output_table_nam
validation_result = plpy.execute(validation_result_query)
plpy.notice("finished validation_result_query, validation_result = " + str(list(validation_result)))
- grp_key_to_best_cp = dict((row['grp_key'], row['cp']) for row in validation_result)
+ grp_key_to_best_cp = dict((""if row['grp_key'] is None else row['grp_key'], row['cp']) for row in validation_result)
# 4) update tree_states to have the best cp cross-validated
for tree in tree_states:
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
index 04f7b82..613d7e6 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
@@ -1659,14 +1659,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_con_splits_transition(
num_splits SMALLINT
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME', 'dst_compute_con_splits_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_con_splits_final(
state MADLIB_SCHEMA.bytea8
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME', 'dst_compute_con_splits_final'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA._dst_compute_con_splits(
@@ -1694,7 +1694,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_entropy_transition(
num_dep_var integer -- constant for the state size
) RETURNS integer[] AS
'MODULE_PATHNAME', 'dst_compute_entropy_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1703,7 +1703,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_entropy_merge(
state2 integer[]
) RETURNS integer[] AS
'MODULE_PATHNAME', 'dst_compute_entropy_merge'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1711,7 +1711,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_entropy_final(
state integer[]
) RETURNS double precision AS
'MODULE_PATHNAME', 'dst_compute_entropy_final'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1738,7 +1738,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._map_catlevel_to_int(
null_as_category BOOLEAN -- flag to check if NULL is treated as a separate category
) RETURNS INTEGER[] AS
'MODULE_PATHNAME', 'map_catlevel_to_int'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -1749,7 +1749,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._initialize_decision_tree(
max_n_surr SMALLINT
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME', 'initialize_decision_tree'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1766,7 +1766,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_leaf_stats_transition(
weights_as_rows BOOLEAN
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME', 'compute_leaf_stats_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1775,7 +1775,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_leaf_stats_merge(
state2 MADLIB_SCHEMA.BYTEA8
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME', 'compute_leaf_stats_merge'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-- One step in the iteration
@@ -1827,7 +1827,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dt_apply(
num_random_features INTEGER
) RETURNS MADLIB_SCHEMA._tree_result_type AS
'MODULE_PATHNAME', 'dt_apply'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------------------------------------------------
@@ -1842,7 +1842,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_surr_stats_transition(
dup_count INTEGER
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME', 'compute_surr_stats_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA._compute_surr_stats(
@@ -1874,7 +1874,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dt_surr_apply(
con_splits MADLIB_SCHEMA.BYTEA8
) RETURNS MADLIB_SCHEMA.BYTEA8 AS
'MODULE_PATHNAME', 'dt_surr_apply'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------------------------------------------------------------------
@@ -1896,7 +1896,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._print_decision_tree(
tree MADLIB_SCHEMA.BYTEA8
) RETURNS MADLIB_SCHEMA._flattened_tree AS
'MODULE_PATHNAME', 'print_decision_tree'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------------------------------------------------------------------
@@ -1906,7 +1906,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_var_importance(
n_con_features INTEGER
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'compute_variable_importance'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------------------------------------------------------------------
@@ -1916,7 +1916,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._predict_dt_response(
con_features DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', 'predict_dt_response'
-LANGUAGE C VOLATILE
+LANGUAGE C VOLATILE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------------------------------------------------------------------
@@ -1926,7 +1926,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._predict_dt_prob(
con_features DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'predict_dt_prob'
-LANGUAGE C VOLATILE
+LANGUAGE C VOLATILE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -2119,7 +2119,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_decision_tree(
verbose BOOLEAN
) RETURNS TEXT
AS 'MODULE_PATHNAME', 'display_decision_tree'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_decision_tree(
@@ -2143,7 +2143,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_decision_tree_surrogate(
cat_n_levels INTEGER[]
) RETURNS TEXT
AS 'MODULE_PATHNAME', 'display_decision_tree_surrogate'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_text_decision_tree(
@@ -2155,7 +2155,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_text_decision_tree(
dependent_levels TEXT[]
) RETURNS TEXT AS
'MODULE_PATHNAME', 'display_text_tree'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -2422,7 +2422,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._prune_and_cplist(
compute_cp_list BOOLEAN
) RETURNS MADLIB_SCHEMA._prune_result_type AS
'MODULE_PATHNAME', 'prune_and_cplist'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -2433,7 +2433,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._convert_to_rpart_format(
n_cats INTEGER
) RETURNS DOUBLE PRECISION[][] AS
'MODULE_PATHNAME', 'convert_to_rpart_format'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -2444,7 +2444,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_split_thresholds(
n_cats integer
) RETURNS double precision[][] AS
'MODULE_PATHNAME', 'get_split_thresholds'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
index fae91bd..209b86a 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
@@ -333,7 +333,7 @@ def forest_train(
if grouping_cols is None:
sql_grp_key_to_grp_cols = """
CREATE TABLE {grp_key_to_grp_cols} AS
- SELECT ''::text AS grp_key, 1 AS gid
+ SELECT ' '::text AS grp_key, 1 AS gid
""".format(**locals())
plpy.notice("sql_grp_key_to_grp_cols:\n" + sql_grp_key_to_grp_cols)
plpy.execute(sql_grp_key_to_grp_cols)
@@ -504,13 +504,13 @@ def forest_train(
is_classification, split_criterion, True,
num_random_features, max_n_surr, null_proxy)
- tree['grp_key'] = ''
+ tree['grp_key'] = ' '
if importance:
tree.update(_compute_var_importance(
schema_madlib, tree,
len(cat_features), len(con_features)))
tree_states = [tree]
- tree_terminated = {'': tree['finished']}
+ tree_terminated = {' ': tree['finished']}
else:
tree_states = _tree_train_grps_using_bins(
schema_madlib, bins, src_view, cat_features, con_features,
@@ -837,7 +837,7 @@ def get_tree(schema_madlib, model_table, gid, sample_id,
SELECT {0}._display_decision_tree(
$1, $2, $3, $4, $5, $6, '{1}', {2}
) as display_tree
- """.format(schema_madlib, "", verbose)
+ """.format(schema_madlib, " ", verbose)
else:
sql_display = """
SELECT {0}._display_text_decision_tree(
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
index 888388c..ae79129 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
@@ -1562,7 +1562,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_bin_value_by_index(
bin_index INTEGER
) RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', 'get_bin_value_by_index'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `');
-----------------------------------------------------------
@@ -1572,7 +1572,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_bin_index_by_value(
feature_index INTEGER
) RETURNS integer AS
'MODULE_PATHNAME', 'get_bin_index_by_value'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `');
------------------------------------------------------------
@@ -1581,7 +1581,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_bin_indices_by_values(
con_splits MADLIB_SCHEMA.bytea8
) RETURNS integer[] AS
'MODULE_PATHNAME', 'get_bin_indices_by_values'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `');
------------------------------------------------------------
@@ -2058,7 +2058,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._convert_to_random_forest_format(
model MADLIB_SCHEMA.bytea8
) RETURNS DOUBLE PRECISION[][] AS
'MODULE_PATHNAME', 'convert_to_random_forest_format'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-- Helper functions for variable importance
@@ -2073,7 +2073,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._rf_cat_imp_score(
cat_feature_distributions DOUBLE PRECISION[][]
) RETURNS DOUBLE PRECISION[][] AS
'MODULE_PATHNAME', 'rf_cat_imp_score'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._rf_con_imp_score(
@@ -2087,7 +2087,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._rf_con_imp_score(
con_index_distrbutions DOUBLE PRECISION[][]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'rf_con_imp_score'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -2096,5 +2096,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalize_sum_array(
target_sum DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'normalize_sum_array'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/regress/clustered_variance.sql_in b/src/ports/postgres/modules/regress/clustered_variance.sql_in
index f05630d..2f7345b 100644
--- a/src/ports/postgres/modules/regress/clustered_variance.sql_in
+++ b/src/ports/postgres/modules/regress/clustered_variance.sql_in
@@ -514,7 +514,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_lin_transition (
coef DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -524,7 +524,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_lin_merge (
state2 MADLIB_SCHEMA.bytea8
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -533,7 +533,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_lin_final (
state MADLIB_SCHEMA.bytea8
) RETURNS MADLIB_SCHEMA.__clustered_agg_result AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -568,7 +568,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_lin_compute_stats (
numRows INTEGER
) RETURNS MADLIB_SCHEMA.__clustered_lin_result AS
'MODULE_PATHNAME', 'clustered_lin_compute_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -753,7 +753,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_log_transition (
coef DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -763,7 +763,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_log_merge (
state2 MADLIB_SCHEMA.bytea8
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -772,7 +772,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_log_final (
state MADLIB_SCHEMA.bytea8
) RETURNS MADLIB_SCHEMA.__clustered_agg_result AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -807,7 +807,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_log_compute_stats (
numRows INTEGER
) RETURNS MADLIB_SCHEMA.__clustered_log_result AS
'MODULE_PATHNAME', 'clustered_log_compute_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -925,7 +925,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_mlog_transition (
ref_cat INTEGER
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -935,7 +935,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_mlog_merge (
state2 MADLIB_SCHEMA.bytea8
) RETURNS MADLIB_SCHEMA.bytea8 AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -944,7 +944,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_mlog_final (
state MADLIB_SCHEMA.bytea8
) RETURNS MADLIB_SCHEMA.__clustered_agg_result AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -982,7 +982,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_mlog_compute_stats (
numRows INTEGER
) RETURNS MADLIB_SCHEMA.__clustered_mlog_result AS
'MODULE_PATHNAME', 'clustered_mlog_compute_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-- DEPRECATED NOTICE -----------------------------------------------------------
diff --git a/src/ports/postgres/modules/regress/linear.sql_in b/src/ports/postgres/modules/regress/linear.sql_in
index afc14d5..b827c43 100644
--- a/src/ports/postgres/modules/regress/linear.sql_in
+++ b/src/ports/postgres/modules/regress/linear.sql_in
@@ -646,7 +646,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_transition(
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -656,14 +656,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_merge_states(
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_final(
state MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.linregr_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-- hetero
@@ -675,7 +675,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_transition(
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -685,7 +685,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_merge_states(
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -694,7 +694,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_final(
state MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.heteroskedasticity_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------------------------------------------------
@@ -810,7 +810,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_predict(
col_ind_var DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME', 'array_dot'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
-- Help messages -------------------------------------------------------
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_predict(
diff --git a/src/ports/postgres/modules/regress/logistic.sql_in b/src/ports/postgres/modules/regress/logistic.sql_in
index e15b416..5fc9ad1 100644
--- a/src/ports/postgres/modules/regress/logistic.sql_in
+++ b/src/ports/postgres/modules/regress/logistic.sql_in
@@ -623,7 +623,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_transition(
DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_cg_step_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -635,7 +635,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_transition(
DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_irls_step_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -647,7 +647,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_transition(
DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_igd_step_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -657,7 +657,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_cg_step_merge_states'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE NOT FENCED STRICT
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -667,7 +667,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_irls_step_merge_states'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE NOT FENCED STRICT
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -677,7 +677,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_igd_step_merge_states'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE NOT FENCED STRICT
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -686,7 +686,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_cg_step_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE NOT FENCED STRICT
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -695,7 +695,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_irls_step_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE NOT FENCED STRICT
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -704,7 +704,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'logregr_igd_step_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE NOT FENCED STRICT
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -776,7 +776,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_distance(
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', 'internal_logregr_cg_step_distance'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -785,7 +785,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.__logregr_result AS
'MODULE_PATHNAME', 'internal_logregr_cg_result'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -795,7 +795,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_distance(
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', 'internal_logregr_irls_step_distance'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -804,7 +804,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.__logregr_result AS
'MODULE_PATHNAME', 'internal_logregr_irls_result'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -814,7 +814,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_distance(
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME', 'internal_logregr_igd_step_distance'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -823,7 +823,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.__logregr_result AS
'MODULE_PATHNAME', 'internal_logregr_igd_result'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------------------
@@ -1044,7 +1044,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_predict(
col_ind_var DOUBLE PRECISION[]
) RETURNS BOOLEAN
AS 'MODULE_PATHNAME', 'logregr_predict'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-- Help messages -------------------------------------------------------
@@ -1081,7 +1081,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_predict_prob(
col_ind_var DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME', 'logregr_predict_prob'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-- Help messages -------------------------------------------------------
diff --git a/src/ports/postgres/modules/regress/marginal.sql_in b/src/ports/postgres/modules/regress/marginal.sql_in
index a19424e..0b459af 100644
--- a/src/ports/postgres/modules/regress/marginal.sql_in
+++ b/src/ports/postgres/modules/regress/marginal.sql_in
@@ -510,7 +510,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.marginal_logregr_step_transition(
coef DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -520,7 +520,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.marginal_logregr_step_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -529,7 +529,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.marginal_logregr_step_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.marginal_logregr_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -785,7 +785,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_transition(
coef DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_merge_states(
@@ -793,7 +793,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -801,7 +801,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.marginal_mlogregr_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1207,7 +1207,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sub_array(
index_array INTEGER[] -- The array containing the index
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-----------------------------------------------------------------------
@@ -1233,7 +1233,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_linregr_int_transition(
derivative DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_linregr_int_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1243,7 +1243,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_linregr_int_merge(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_linregr_int_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1252,7 +1252,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_linregr_int_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.margins_result
AS 'MODULE_PATHNAME', 'margins_linregr_int_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1299,7 +1299,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_logregr_int_transition(
x_unset DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_logregr_int_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1309,7 +1309,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_logregr_int_merge(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_logregr_int_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1318,7 +1318,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_logregr_int_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.margins_result
AS 'MODULE_PATHNAME', 'margins_logregr_int_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1375,7 +1375,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_mlogregr_int_transition(
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_mlogregr_int_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1385,7 +1385,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_mlogregr_int_merge(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_mlogregr_int_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1394,7 +1394,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_mlogregr_int_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.margins_result
AS 'MODULE_PATHNAME', 'margins_mlogregr_int_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1450,7 +1450,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_coxph_int_transition(
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_coxph_int_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1460,7 +1460,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_coxph_int_merge(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'margins_coxph_int_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1469,7 +1469,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_coxph_int_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.margins_result
AS 'MODULE_PATHNAME', 'margins_coxph_int_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------------
@@ -1479,7 +1479,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_compute_stats(
std_err DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.margins_result
AS 'MODULE_PATHNAME', 'margins_compute_stats'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
diff --git a/src/ports/postgres/modules/regress/multilogistic.sql_in b/src/ports/postgres/modules/regress/multilogistic.sql_in
index a4acf60..1f4f92b 100644
--- a/src/ports/postgres/modules/regress/multilogistic.sql_in
+++ b/src/ports/postgres/modules/regress/multilogistic.sql_in
@@ -523,7 +523,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_irls_step_transition
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -534,7 +534,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_irls_step_merge_states
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -544,7 +544,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_irls_step_final
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -575,14 +575,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__internal_mlogregr_irls_step_distance(
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__internal_mlogregr_irls_result(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.mlogregr_result AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -590,7 +590,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__internal_mlogregr_summary_results(
/*+ state */ DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.mlogregr_summary_result AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
/**
@@ -1071,7 +1071,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_format(
ref_category INTEGER
) RETURNS SETOF MADLIB_SCHEMA.__mlogregr_cat_coef
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -1084,7 +1084,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_predict_prob(
col_ind_var DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'mlogregr_predict_prob'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -1097,7 +1097,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_predict_response(
col_ind_var DOUBLE PRECISION[]
) RETURNS INTEGER AS
'MODULE_PATHNAME', 'mlogregr_predict_response'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/regress/robust.sql_in b/src/ports/postgres/modules/regress/robust.sql_in
index 7964eb0..c5ecb7b 100644
--- a/src/ports/postgres/modules/regress/robust.sql_in
+++ b/src/ports/postgres/modules/regress/robust.sql_in
@@ -531,7 +531,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_linregr_transition(
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -542,7 +542,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_linregr_merge_states(
RETURNS MADLIB_SCHEMA.bytea8
AS 'MODULE_PATHNAME'
LANGUAGE C
-IMMUTABLE STRICT
+IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -551,7 +551,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_linregr_final(
state MADLIB_SCHEMA.bytea8)
RETURNS MADLIB_SCHEMA.robust_linregr_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -646,7 +646,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_logregr_step_transition(
coef DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -656,7 +656,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_logregr_step_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -665,7 +665,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_logregr_step_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.robust_logregr_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -864,7 +864,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_robust_step_transition
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -876,7 +876,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_robust_step_merge_states
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
@@ -887,7 +887,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_robust_step_final
)
RETURNS MADLIB_SCHEMA.robust_mlogregr_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------
diff --git a/src/ports/postgres/modules/sample/sample.sql_in b/src/ports/postgres/modules/sample/sample.sql_in
index 8f8a56f..3652f4c 100644
--- a/src/ports/postgres/modules/sample/sample.sql_in
+++ b/src/ports/postgres/modules/sample/sample.sql_in
@@ -69,7 +69,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_transition_int64(
AS 'MODULE_PATHNAME'
LANGUAGE C
VOLATILE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -80,7 +80,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_merge_int64(
AS 'MODULE_PATHNAME'
LANGUAGE C
VOLATILE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -90,7 +90,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_final_int64(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -128,7 +128,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_transition_vector(
AS 'MODULE_PATHNAME'
LANGUAGE C
VOLATILE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -139,7 +139,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_merge_vector(
AS 'MODULE_PATHNAME'
LANGUAGE C
VOLATILE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -149,7 +149,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_final_vector(
AS 'MODULE_PATHNAME'
LANGUAGE C
IMMUTABLE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -173,7 +173,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_random(
AS 'MODULE_PATHNAME'
LANGUAGE C
VOLATILE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------------------------------------------------
@@ -183,7 +183,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_random(
AS 'MODULE_PATHNAME'
LANGUAGE C
VOLATILE
-STRICT
+STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
---------------------------------------------------------------------------
@@ -191,5 +191,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.index_weighted_sample(
double precision[]
) RETURNS integer
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/sample/stratified_sample.py_in b/src/ports/postgres/modules/sample/stratified_sample.py_in
index 90d12d5..7fe88f0 100644
--- a/src/ports/postgres/modules/sample/stratified_sample.py_in
+++ b/src/ports/postgres/modules/sample/stratified_sample.py_in
@@ -88,9 +88,9 @@ def stratified_sample(schema_madlib, source_table, output_table, proportion,
# Find the cut-off label for the given proportion
sql2 = """ CREATE TEMP TABLE {perc} AS (
- SELECT {grouping_cols}, percentile_disc({proportion})
- WITHIN GROUP (ORDER BY __samp_out_label) AS __samp_out_label
- FROM {label} GROUP BY {grouping_cols})""".format(**locals())
+ SELECT {grouping_cols},
+ {schema_madlib}.percentile_disc(array_agg(__samp_out_label),{proportion}) AS __samp_out_label
+ FROM {label} where __samp_out_label is not null GROUP BY {grouping_cols})""".format(**locals())
plpy.execute(sql2)
# Select every record that has a label under the threshold
diff --git a/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in b/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in
index 58d6e8e..4475071 100644
--- a/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in
+++ b/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in
@@ -55,7 +55,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_a_b_transition(
/* H values */ DOUBLE PRECISION[],
/* S */ DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -65,7 +65,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_a_b_merge(
/* left state */ DOUBLE PRECISION[],
/* right state */ DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -81,7 +81,7 @@ CREATE TYPE MADLIB_SCHEMA.__coxph_a_b_result AS(
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_a_b_final(
/* right state */ DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__coxph_a_b_result AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -117,7 +117,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_compute_w(
A DOUBLE PRECISION,
B DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
----------------------------------------------------------------------
@@ -136,7 +136,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_compute_clustered_stats(
hessian DOUBLE PRECISION[],
A DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__coxph_cl_var_result AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/stats/correlation.sql_in b/src/ports/postgres/modules/stats/correlation.sql_in
index 4932385..b9b6f9d 100644
--- a/src/ports/postgres/modules/stats/correlation.sql_in
+++ b/src/ports/postgres/modules/stats/correlation.sql_in
@@ -396,7 +396,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_transition(
mean double precision[]
) RETURNS double precision[] AS
'MODULE_PATHNAME', 'correlation_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_merge(
@@ -404,14 +404,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_merge(
right_state double precision[]
) RETURNS double precision[] AS
'MODULE_PATHNAME', 'correlation_merge_states'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_final(
state double precision[]
) RETURNS double precision[] AS
'MODULE_PATHNAME', 'correlation_final'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.correlation_agg(
diff --git a/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in b/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in
index e8faa01..b3b70b6 100644
--- a/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in
+++ b/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in
@@ -589,7 +589,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_transition(
use_abs BOOLEAN
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_merge(
@@ -597,14 +597,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_merge(
right DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_final(
state DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.array_avg(
@@ -811,7 +811,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._coxph_predict_resp(
pred_type TEXT
) RETURNS DOUBLE PRECISION
AS 'MODULE_PATHNAME', 'coxph_predict_resp'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -821,7 +821,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._coxph_predict_terms(
mean_ind_var DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'coxph_predict_terms'
-LANGUAGE C STRICT IMMUTABLE
+LANGUAGE C STRICT IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -858,7 +858,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._split_transition(
/* num_splits */ INTEGER
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'split_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
-------------
@@ -868,7 +868,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._split_merge(
/* right state */ DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'split_merge'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------
@@ -877,7 +877,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._split_final(
/* final state */ DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'split_final'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------
@@ -908,7 +908,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_grpid(
reverse BOOLEAN
) RETURNS INTEGER AS
'MODULE_PATHNAME', 'compute_grpid'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------
@@ -948,7 +948,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.compute_coxph_result(
stds DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.coxph_result AS
'MODULE_PATHNAME', 'compute_coxph_result'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -968,7 +968,7 @@ MADLIB_SCHEMA.coxph_improved_step_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.coxph_step_result AS
'MODULE_PATHNAME', 'coxph_improved_step_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -984,7 +984,7 @@ MADLIB_SCHEMA.coxph_improved_step_transition(
)
RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'coxph_improved_step_transition'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------
@@ -1035,7 +1035,7 @@ DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.coxph_improved_strata_step_inner(
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_step_inner_final(
state DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
----------------------------------------------------------------------
@@ -1044,7 +1044,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_step_outer_transition(
/*+ state1 */ DOUBLE PRECISION[],
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------------------------------
@@ -1071,7 +1071,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_improved_strata_step_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.coxph_step_result AS
'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
------------------------------------
@@ -1149,7 +1149,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__zph_transition(
/*+ coef */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'zph_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1159,7 +1159,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__zph_merge(
/*+ right_state */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'zph_merge'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1167,7 +1167,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__zph_final(
/*+ left_state */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'zph_final'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1199,7 +1199,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_scale_resid(
resid DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'coxph_scale_resid'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1213,7 +1213,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_elem_corr_transition(
/*+ y */ DOUBLE PRECISION)
RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'array_elem_corr_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1223,7 +1223,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_elem_corr_merge(
/*+ right_state */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'array_elem_corr_merge'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1231,7 +1231,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_elem_corr_final(
/*+ state */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'array_elem_corr_final'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1265,7 +1265,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_resid_stat_transition(
/*+ m */ INTEGER)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'coxph_resid_stat_transition'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1275,7 +1275,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_resid_stat_merge(
/*+ state2 */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'coxph_resid_stat_merge'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1292,7 +1292,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_resid_stat_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.__cox_resid_stat_result
AS 'MODULE_PATHNAME', 'coxph_resid_stat_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1479,7 +1479,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_element_min(
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'array_element_min'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -1503,7 +1503,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_element_max(
)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME', 'array_element_max'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/stats/distribution.sql_in b/src/ports/postgres/modules/stats/distribution.sql_in
index 30ee3cd..a60ccb8 100644
--- a/src/ports/postgres/modules/stats/distribution.sql_in
+++ b/src/ports/postgres/modules/stats/distribution.sql_in
@@ -14,14 +14,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.vectorized_distribution_transition(
levels integer[]
) RETURNS double precision[][] AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.vectorized_distribution_final(
state double precision[][]
) RETURNS double precision[][] AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.vectorized_distribution_agg(
@@ -45,14 +45,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.discrete_distribution_transition(
level integer
) RETURNS double precision[] AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE
+LANGUAGE c IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.discrete_distribution_final(
state double precision[]
) RETURNS double precision[] AS
'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `');
DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.discrete_distribution_agg(
diff --git a/src/ports/postgres/modules/stats/hypothesis_tests.sql_in b/src/ports/postgres/modules/stats/hypothesis_tests.sql_in
index 61dbb89..ba43a03 100644
--- a/src/ports/postgres/modules/stats/hypothesis_tests.sql_in
+++ b/src/ports/postgres/modules/stats/hypothesis_tests.sql_in
@@ -572,7 +572,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_one_transition(
value DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_merge_states(
@@ -580,7 +580,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
@@ -588,7 +588,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_one_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.t_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
DROP TYPE IF EXISTS MADLIB_SCHEMA.f_test_result CASCADE;
@@ -604,7 +604,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.f_test_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.f_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
@@ -664,7 +664,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_two_transition(
"value" DOUBLE PRECISION)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
@@ -672,7 +672,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_two_pooled_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.t_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
* @brief Perform two-sample pooled (i.e., equal variances) Student t-test
@@ -739,7 +739,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_two_unpooled_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.t_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
* @brief Perform unpooled (i.e., unequal variances) t-test (also known as
@@ -861,7 +861,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition(
df BIGINT
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition(
@@ -870,7 +870,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition(
expected DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition(
@@ -878,7 +878,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition(
observed BIGINT
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_merge_states(
@@ -886,7 +886,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
DROP TYPE IF EXISTS MADLIB_SCHEMA.chi2_test_result CASCADE;
@@ -902,7 +902,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_final(
state DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.chi2_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
@@ -1009,7 +1009,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.ks_test_transition(
"numSecond" BIGINT
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
DROP TYPE IF EXISTS MADLIB_SCHEMA.ks_test_result CASCADE;
@@ -1023,7 +1023,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.ks_test_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.ks_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
@@ -1097,7 +1097,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mw_test_transition(
"value" DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
DROP TYPE IF EXISTS MADLIB_SCHEMA.mw_test_result CASCADE;
@@ -1112,7 +1112,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mw_test_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.mw_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
* @brief Perform Mann-Whitney test
@@ -1184,7 +1184,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_transition(
"precision" DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_transition(
@@ -1192,7 +1192,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_transition(
value DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
@@ -1211,7 +1211,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.wsr_test_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
* @brief Perform Wilcoxon-Signed-Rank test
@@ -1336,7 +1336,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_transition(
value DOUBLE PRECISION)
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_merge_states(
@@ -1344,14 +1344,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_merge_states(
state2 DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_final(
state DOUBLE PRECISION[])
RETURNS MADLIB_SCHEMA.one_way_anova_result
AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!NO SQL!>, <!!>);
/**
* @brief Perform one-way analysis of variance
diff --git a/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in b/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in
index 5bc58f7..710a649 100644
--- a/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in
+++ b/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in
@@ -60,7 +60,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_coxph_step_transition(
/* H values */ DOUBLE PRECISION[],
/* S */ DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -69,7 +69,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_coxph_step_final(
/* state */ DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__rb_coxph_result AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------
@@ -106,7 +106,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_h_s_transition(
/* x */ DOUBLE PRECISION[],
/* coef */ DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------
@@ -115,7 +115,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_h_s_merge(
/*+ left_state */ DOUBLE PRECISION[],
/*+ right_state */ DOUBLE PRECISION[])
RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------
@@ -123,7 +123,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_h_s_final(
/* state */ DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__rb_coxph_hs_result AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------
@@ -148,7 +148,7 @@ CREATE AGGREGATE MADLIB_SCHEMA.coxph_h_s(
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_coxph_strata_step_final(
/* state */ DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------
@@ -185,7 +185,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_sum_strata_transition(
in_state1 DOUBLE PRECISION[],
in_state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------
@@ -193,7 +193,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_sum_strata_final(
in_state DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__rb_coxph_result AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
--------------------------------
diff --git a/src/ports/postgres/modules/summary/Summarizer.py_in b/src/ports/postgres/modules/summary/Summarizer.py_in
index 5790972..a4030d3 100644
--- a/src/ports/postgres/modules/summary/Summarizer.py_in
+++ b/src/ports/postgres/modules/summary/Summarizer.py_in
@@ -169,17 +169,19 @@ class Summarizer:
return "NULL"
def mfv_type(get_count, c):
- slicing = ('0:0', '1:1')[get_count]
+ slicing = ('0,0', '1,1')[get_count]
mfv_method = ('mfvsketch_top_histogram',
'mfvsketch_quick_histogram')[self._get_mfv_quick]
+ dim_end = self._how_many_mfv - 1
return """
array_to_string(({schema_madlib}.{mfv_method}(
- {column},{topk}))[0:{topk}-1][{slice}],
+ {column},{topk}))[0,{dim_end}][{slice}],
'{delimiter}')""".format(
schema_madlib=self._schema_madlib,
mfv_method=mfv_method,
column=c['attname'],
topk=self._how_many_mfv,
+ dim_end=dim_end,
slice=slicing,
delimiter=self._delimiter)
# ------ End of Helper sub-functions ------
diff --git a/src/ports/postgres/modules/svm/svm.sql_in b/src/ports/postgres/modules/svm/svm.sql_in
index ba05e86..8a538aa 100644
--- a/src/ports/postgres/modules/svm/svm.sql_in
+++ b/src/ports/postgres/modules/svm/svm.sql_in
@@ -1155,20 +1155,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linear_svm_igd_transition(
tuple_weight double precision
)
RETURNS double precision[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linear_svm_igd_merge(
state1 double precision[],
state2 double precision[])
RETURNS double precision[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linear_svm_igd_final(
state double precision[])
RETURNS double precision[] AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
/**
@@ -1213,13 +1213,13 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_linear_svm_igd_distance(
/*+ state1 */ double precision[],
/*+ state2 */ double precision[])
RETURNS double precision AS 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_linear_svm_igd_result(
/*+ state */ double precision[])
RETURNS MADLIB_SCHEMA.linear_svm_result AS 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL');
diff --git a/src/ports/postgres/modules/tsa/arima.py_in b/src/ports/postgres/modules/tsa/arima.py_in
index 6ae7588..2dfea47 100644
--- a/src/ports/postgres/modules/tsa/arima.py_in
+++ b/src/ports/postgres/modules/tsa/arima.py_in
@@ -202,7 +202,7 @@ def arima_train (schema_madlib, input_table, output_table,
# Also add an auxiliary array column which contains p previous time
# series values.
# tbl_ts is the time series table we are going to use. For convenience,
- # we fix the column names: tid, distid and tvals for time, distribution
+ # we fix the column names: timeid, distid and tvals for time, distribution
# id and an array of current value and p previous values.
# if the data has been differenced, we only need to do adjustment
if d > 0:
@@ -621,16 +621,16 @@ def _redist_data(schema_madlib, input_table, timestamp_col,
dist_table = unique_string() + "redist__"
# Redistribute the data
- # <tid, tval, distid>
+ # <timeid, tval, distid>
plpy.execute(
"""
create temp table {dist_table} as
select
- ((tid - 1) / {chunk_size})::integer + 1 as distid,
- tid, origin_tid, tval
+ ((timeid - 1) / {chunk_size})::integer + 1 as distid,
+ timeid, origin_tid, tval
from (
select
- row_number() over (order by {timestamp_col}) as tid,
+ row_number() over (order by {timestamp_col}) as timeid,
{timestamp_col} as origin_tid,
{timeseries_col} as tval
from
@@ -650,10 +650,10 @@ def _redist_data(schema_madlib, input_table, timestamp_col,
"""
insert into {dist_table}
select
- o.distid + 1, tid, NULL, tval
+ o.distid + 1, timeid, NULL, tval
from (
select
- distid, max(tid) maxid
+ distid, max(timeid) maxid
from
{dist_table}
where
@@ -662,24 +662,24 @@ def _redist_data(schema_madlib, input_table, timestamp_col,
) q1, {dist_table} o
where
q1.distid = o.distid and
- q1.maxid - o.tid < {d}
+ q1.maxid - o.timeid < {d}
""".format(dist_table=dist_table, d=d))
output_table = unique_string() + "redist_agg__"
# Aggregate the chunk into an array
# Note that each aggregated chunk except for the first one
- # has additonal p or d elements and tid is 1 for the first
- # chunk and is the actual starting tid for the other chunks
+ # has additonal p or d elements and timeid is 1 for the first
+ # chunk and is the actual starting timeid for the other chunks
# - the corresponding tval is the pth (0 based) element in
# tvals
- # Note that both distid and tid starts from 1
+ # Note that both distid and timeid starts from 1
plpy.execute(
"""
create temp table {output_table} as
select
distid,
- (distid - 1) * {chunk_size} + 1 as tid,
- array_agg(tval order by tid) as tvals
+ (distid - 1) * {chunk_size} + 1 as timeid,
+ array_agg(tval order by timeid) as tvals
from
{dist_table}
group by
@@ -722,14 +722,14 @@ def _diff (schema_madlib, input_table, timestamp_col, timeseries_col, d,
create temp table {output_table} as
select
distid::integer,
- case when tid = 1 then 1::integer
- else (tid - {d})::integer
- end as tid,
+ case when timeid = 1 then 1::integer
+ else (timeid - {d})::integer
+ end as timeid,
tvals
from
(
select
- distid, tid,
+ distid, timeid,
{schema_madlib}.__arima_diff(tvals, {d}) tvals
from
{input_table}
@@ -759,7 +759,7 @@ def _adjust_diff_data(schema_madlib, diff_table, p):
"""
create temp table {output_table} as
select
- t1.distid::integer, t1.tid::integer,
+ t1.distid::integer, t1.timeid::integer,
{schema_madlib}.__arima_adjust(t1.distid, t1.tvals, t2.tvals, {p}) as tvals
from
{diff_table} as t1, {diff_table} as t2
@@ -816,7 +816,7 @@ def _create_resid_tbl (schema_madlib, tbl_ts, p, d, q, phi,
if q != 0:
theta_str = "array[" + ",".join(str(i) for i in theta) + "]"
- # Joining on tid is really slow on large data
+ # Joining on timeid is really slow on large data
# (more than 100 times slower).
# This is why we use join on distid, which gives almost the
# same performance as that of the original code.
@@ -1099,7 +1099,7 @@ def _compute_statistics (schema_madlib, tbl_ts, p, q, phi, theta,
Args:
@param schema_madlib - MADlib schema name
- @param tbl_ts - time series table name, it has two columns: tid
+ @param tbl_ts - time series table name, it has two columns: timeid
and tvals, which is the p previous values plus the
current value ordered by time.
@param p - integer, AR order
@@ -1129,7 +1129,7 @@ def _compute_statistics (schema_madlib, tbl_ts, p, q, phi, theta,
'{theta}'::double precision[],
{mean},
{delta}
- order by tid
+ order by timeid
) as f
from {tbl_ts}
) s
diff --git a/src/ports/postgres/modules/tsa/arima.sql_in b/src/ports/postgres/modules/tsa/arima.sql_in
index 48f0abd..5769ca3 100644
--- a/src/ports/postgres/modules/tsa/arima.sql_in
+++ b/src/ports/postgres/modules/tsa/arima.sql_in
@@ -641,7 +641,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_residual
prez DOUBLE PRECISION[]
)
RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_residual'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -652,7 +652,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_diff
d INTEGER
)
RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_diff'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -666,7 +666,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_adjust
p INTEGER
)
RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_adjust'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -680,7 +680,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_delta
u DOUBLE PRECISION -- Weight of gradient-descent step
)
RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_lm_delta'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -713,7 +713,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm
)
RETURNS MADLIB_SCHEMA.__arima_lm_result
AS 'MODULE_PATHNAME', 'arima_lm'
-LANGUAGE C
+LANGUAGE C NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -728,7 +728,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_result_sfunc (
z2 DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'arima_lm_result_sfunc'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -739,7 +739,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_result_pfunc (
state2 DOUBLE PRECISION[]
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'arima_lm_result_pfunc'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -759,7 +759,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_result_ffunc (
state_data DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__arima_lm_sum_result AS
'MODULE_PATHNAME', 'arima_lm_result_ffunc'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -799,7 +799,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_stat_sfunc (
delta DOUBLE PRECISION
) RETURNS DOUBLE PRECISION[] AS
'MODULE_PATHNAME', 'arima_lm_stat_sfunc'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
@@ -818,7 +818,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_stat_ffunc (
state_data DOUBLE PRECISION[]
) RETURNS MADLIB_SCHEMA.__arima_lm_stat_result AS
'MODULE_PATHNAME', 'arima_lm_stat_ffunc'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/utilities/control.py_in b/src/ports/postgres/modules/utilities/control.py_in
index 32f2f59..3965e50 100644
--- a/src/ports/postgres/modules/utilities/control.py_in
+++ b/src/ports/postgres/modules/utilities/control.py_in
@@ -234,7 +234,7 @@ class AOControl(ContextDecorator):
def __init__(self, enable=False):
self.to_enable = enable
self.was_ao_enabled = False
- self.guc_exists = True
+ self.guc_exists = False
self.storage_options_dict = dict()
def _parse_gp_default_storage_options(self, gp_default_storage_options_str):
diff --git a/src/ports/postgres/modules/utilities/in_mem_group_control.py_in b/src/ports/postgres/modules/utilities/in_mem_group_control.py_in
index 86f0c17..be28ea0 100644
--- a/src/ports/postgres/modules/utilities/in_mem_group_control.py_in
+++ b/src/ports/postgres/modules/utilities/in_mem_group_control.py_in
@@ -106,10 +106,14 @@ class BaseState(object):
if col_grp_state == '':
self._is_none = True
for s in ret_states:
+ if s[col_grp_key] is None:
+ s[col_grp_key] = ' '
self._state[s[col_grp_key]] = None
return failed_grp_keys
for t in ret_states:
_grp_key, _grp_state = t[col_grp_key], t[col_grp_state]
+ if _grp_key is None:
+ _grp_key = ' '
if _grp_state is None:
failed_grp_keys.append(_grp_key)
else:
@@ -204,6 +208,7 @@ class GroupIterationController:
col_n_tuples: Name of the column containing count of tuples.
Set to a unique string if not defined
"""
+ NULL_list = ['Null', 'NULL','None','NONE']
self.schema_madlib = arg_dict['schema_madlib']
self.in_with = False
self.iteration = -1
@@ -222,6 +227,8 @@ class GroupIterationController:
if arg_dict["grouping_col"] is None
else arg_dict["grouping_col"]),
)
+ if self.kwargs['grouping_str'] in NULL_list:
+ self.kwargs['grouping_str'] = "' '"
self.grp_to_n_tuples = {}
self.failed_grp_keys = []
@@ -287,6 +294,7 @@ class GroupIterationController:
else "{grouping_col}").format(**self.kwargs)
groupby_str = ("{col_grp_null}" if self.is_group_null
else "{grouping_col}").format(**self.kwargs)
+
plpy.execute("""
DROP TABLE IF EXISTS {rel_state};
CREATE TEMPORARY TABLE {rel_state} AS (
@@ -384,6 +392,7 @@ class GroupIterationController:
select_rel_state=group_param.select_rel_state,
select_n_tuples=group_param.select_n_tuples,
**self.kwargs)
+
insert_plan = plpy.prepare(insert_sql,
["text[]", group_param.grouped_state_type,
"text[]", "bigint[]"])
diff --git a/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in b/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in
index c25463c..9346d4a 100644
--- a/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in
+++ b/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in
@@ -284,8 +284,8 @@ class MiniBatchPreProcessor:
{self.schema_madlib}.matrix_agg({dep_colname}) as {dep_colname},
{self.schema_madlib}.matrix_agg({ind_colname}) as {ind_colname}
FROM (
- SELECT (row_number() OVER ({partition_by} ORDER BY random()) - 1)
- / {buffer_size}
+ SELECT trunc((row_number() OVER ({partition_by} ORDER BY random()) - 1)
+ / {buffer_size})
as {row_id}, *
FROM (
{standardize_query}
@@ -474,7 +474,7 @@ class MiniBatchBufferSizeCalculator:
default_buffer_size to int, because it will be used to calculate the
row id of the packed input. The query looks like this
- SELECT (row_number() OVER (ORDER BY random()) - 1) / {buffer_size}
+ SELECT (row_number() OVER (ORDER BY random())) / {buffer_size}
This calculation has to return an int for which buffer_size has
to be an int
diff --git a/src/ports/postgres/modules/utilities/path.sql_in b/src/ports/postgres/modules/utilities/path.sql_in
index 6c102eb..6ab554a 100644
--- a/src/ports/postgres/modules/utilities/path.sql_in
+++ b/src/ports/postgres/modules/utilities/path.sql_in
@@ -551,7 +551,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.path_pattern_match
overlapping_patterns BOOLEAN
) RETURNS MADLIB_SCHEMA.path_match_result
AS 'MODULE_PATHNAME', 'path_pattern_match'
-LANGUAGE C STRICT
+LANGUAGE C STRICT NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/utilities/pivot.py_in b/src/ports/postgres/modules/utilities/pivot.py_in
index 1b1a5d9..48a8376 100644
--- a/src/ports/postgres/modules/utilities/pivot.py_in
+++ b/src/ports/postgres/modules/utilities/pivot.py_in
@@ -305,8 +305,7 @@ def pivot(schema_madlib, source_table, output_table, index, pivot_cols,
# Aggregate over each pivot column, while filtering all NULL
# values created by previous query.
sub_pivot_str_sel = _fill_value_wrapper(
- "{agg}({p_name}) "
- " FILTER (WHERE {p_name} IS NOT NULL)".
+ "{agg}(CASE WHEN {p_name} IS NOT NULL THEN {p_name} END )".
format(agg=agg, p_name=p_name))
if not is_array_output:
# keep spaces around the 'AS'
diff --git a/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in b/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in
index b6b2996..3f7d0f1 100644
--- a/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in
+++ b/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in
@@ -139,7 +139,7 @@ SELECT assert
) from (select * from minibatch_preprocessing_out_summary) summary;
-- grouping with the same dataset
-\set expected_grouping_row_count '\'' 1,1,2 '\''
+\set expected_grouping_row_count '''1,1,2'''
DROP TABLE IF EXISTS minibatch_preprocessing_out, minibatch_preprocessing_out_standardization, minibatch_preprocessing_out_summary;
SELECT minibatch_preprocessor('minibatch_preprocessing_input', 'minibatch_preprocessing_out', 'length>0.2', 'ARRAY[diameter,height,whole,shucked,viscera,shell]', 'rings', 4);
SELECT assert
@@ -231,8 +231,8 @@ INSERT INTO minibatch_preprocessing_input(x1,x2,y) VALUES
SELECT minibatch_preprocessor('minibatch_preprocessing_input', 'minibatch_preprocessing_out', 'y', 'ARRAY[x1,x2]', NULL, 2);
-- since the order is not deterministic, we assert for all possible orders
-\set expected_normalized_independent_var1 '\'' {{-1, -1},{1, 1}} '\''
-\set expected_normalized_independent_var2 '\'' {{1, 1},{-1, -1}} '\''
+\set expected_normalized_independent_var1 '''{{-1,-1},{1,1}}'''
+\set expected_normalized_independent_var2 '''{{1,1},{-1,-1}}'''
SELECT assert
(
diff --git a/src/ports/postgres/modules/utilities/test/pivot.sql_in b/src/ports/postgres/modules/utilities/test/pivot.sql_in
index fb034e7..695805a 100644
--- a/src/ports/postgres/modules/utilities/test/pivot.sql_in
+++ b/src/ports/postgres/modules/utilities/test/pivot.sql_in
@@ -48,8 +48,8 @@ SELECT assert(val_avg_piv_20 = 3, 'Wrong output in pivoting') FROM pivout WHERE
DROP VIEW IF EXISTS pivset_ext;
CREATE VIEW pivset_ext AS
SELECT *,
- COALESCE(id + ("PIVSET".val / 3), 0) AS id2,
- COALESCE(piv + ("PIVSET".val / 3), 0) AS piv2,
+ COALESCE(id + floor("PIVSET".val / 3), 0)::int AS id2,
+ COALESCE(piv + floor("PIVSET".val / 3), 0)::int AS piv2,
COALESCE(val + 10, 0) AS val2
FROM "PIVSET";
SELECT id,id2,piv,piv2,val,val2 FROM pivset_ext
diff --git a/src/ports/postgres/modules/utilities/test/sessionize.sql_in b/src/ports/postgres/modules/utilities/test/sessionize.sql_in
index 808970c..1d948f3 100644
--- a/src/ports/postgres/modules/utilities/test/sessionize.sql_in
+++ b/src/ports/postgres/modules/utilities/test/sessionize.sql_in
@@ -75,7 +75,7 @@ INSERT INTO eventlog_installchk VALUES
(to_timestamp(NULL, 'MM/DD/YYYY HH:MI:SS'), 103711, NULL, 'WINE', 0, 1, 't'),
(to_timestamp('04/15/2016 02:17:00', 'MM/DD/YYYY HH:MI:SS'), 103711, 1, 'BEER', 0, 1, 't'),
(to_timestamp('04/15/2016 02:21:00', 'MM/DD/YYYY HH:MI:SS'), 103711, 2, 'LANDING', 0, 1, 't'),
-(to_timestamp('04/15/2016 02:31:0.05', 'MM/DD/YYYY HH:MI:SS'), 103711, 3, 'WINE', 0, 1, 't');
+(to_timestamp('04/15/2016 02:31:0.05', 'MM/DD/YYYY HH:MI:SS.FF'), 103711, 3, 'WINE', 0, 1, 't');
SELECT sessionize(
diff --git a/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in b/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in
index b43b39f..860d20d 100644
--- a/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in
+++ b/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in
@@ -397,7 +397,7 @@ SELECT vec2cols(
'out_table2',
'feature_vector'
);
-SELECT assert((select data_type from information_schema.columns where table_name = 'out_table2' limit 1) = 'double precision', 'Incorrect type casting of features for cols2vec');
+SELECT assert((select data_type from information_schema.columns where table_name = 'out_table2' limit 1) = 'numeric', 'Incorrect type casting of features for cols2vec');
DROP TABLE IF EXISTS out_table, out_table_summary, out_table2;
SELECT cols2vec(
diff --git a/src/ports/postgres/modules/utilities/utilities.py_in b/src/ports/postgres/modules/utilities/utilities.py_in
index 577235e..6941926 100644
--- a/src/ports/postgres/modules/utilities/utilities.py_in
+++ b/src/ports/postgres/modules/utilities/utilities.py_in
@@ -611,6 +611,8 @@ def set_client_min_messages(new_level):
def is_pg_major_version_less_than(schema_madlib, compare_version, **kwargs):
version = plpy.execute("select version()")[0]["version"]
+ if "openGauss" in version:
+ return True
regex = re.compile('PostgreSQL\s*([0-9]+)([0-9.beta]+)', re.IGNORECASE)
version = regex.findall(version)
if len(version) > 0 and int(version[0][0]) < compare_version:
diff --git a/src/ports/postgres/modules/utilities/utilities.sql_in b/src/ports/postgres/modules/utilities/utilities.sql_in
index bbf861d..2d938e6 100644
--- a/src/ports/postgres/modules/utilities/utilities.sql_in
+++ b/src/ports/postgres/modules/utilities/utilities.sql_in
@@ -265,11 +265,11 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `READS SQL DATA', `');
* >PostgreSQL documentation</a>. For portability, MADlib code should not make
* use of such "features" directly, but only use isnan() instead.
*
- * @param number
- * @returns \c TRUE if \c number is \c NaN, \c FALSE otherwise
+ * @param number1
+ * @returns \c TRUE if \c number1 is \c NaN, \c FALSE otherwise
*/
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.isnan(
- number DOUBLE PRECISION
+ number1 DOUBLE PRECISION
) RETURNS BOOLEAN
AS $$
SELECT $1 = 'NaN'::DOUBLE PRECISION;
@@ -313,7 +313,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.noop()
RETURNS VOID
AS 'MODULE_PATHNAME'
LANGUAGE c
-VOLATILE
+VOLATILE NOT FENCED
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
------------------------------------------------------------------------
@@ -434,7 +434,7 @@ $$m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__to_char(val anyelement)
RETURNS TEXT
AS 'MODULE_PATHNAME', '__to_text'
-LANGUAGE C STRICT IMMUTABLE;
+LANGUAGE C STRICT IMMUTABLE NOT FENCED;
------------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/validate_args.py_in b/src/ports/postgres/modules/utilities/validate_args.py_in
index a21be58..6f2f3bd 100644
--- a/src/ports/postgres/modules/utilities/validate_args.py_in
+++ b/src/ports/postgres/modules/utilities/validate_args.py_in
@@ -215,10 +215,11 @@ def get_first_schema(table_name):
def drop_tables(table_list):
"""
Drop tables specified in table_list.
+ Notice: Both TEMP and non-TEMP objects maybe in the table_list at the same time.
"""
- drop_str = ', '.join(table_list)
- if drop_str:
- plpy.execute("DROP TABLE IF EXISTS {0}".format(drop_str))
+ if len(table_list) > 0:
+ for table in table_list:
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(table))
def table_is_empty(tbl, filter_str=None):
diff --git a/src/ports/postgres/modules/xgboost_gs/__init__.py_in b/src/ports/postgres/modules/xgboost_gs/__init__.py_in
new file mode 100644
index 0000000..e69de29
diff --git a/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_classifi.sql_in b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_classifi.sql_in
new file mode 100644
index 0000000..4446481
--- /dev/null
+++ b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_classifi.sql_in
@@ -0,0 +1,257 @@
+/*
+ * Test case for sklearn-api xgboost.
+ */
+
+m4_include(`SQLCommon.m4')
+
+select MADLIB_SCHEMA.xgboost_sk_Classifier();
+select MADLIB_SCHEMA.xgboost_sk_Classifier('usage');
+
+DROP TABLE IF EXISTS iris;
+create table iris (id serial, a float, d float, c float, b float, label int);
+
+INSERT into iris (a, b, c, d, label) values (5.1, 3.5, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.9, 3.0, 1.4, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (4.7, 3.2, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.6, 3.1, 1.5, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (5.0, 3.6, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.4, 3.9, 1.7, 0.4, 0);
+INSERT into iris (a, b, c, d, label) values (4.6, 3.4, 1.4, 0.3, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.4, 1.5, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (4.4, 2.9, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.9, 3.1, 1.5, 0.1, 0);
+INSERT into iris (a, b, c, d, label) values (5.4, 3.7, 1.5, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.8, 3.4, 1.6, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (4.8, 3.0, 1.4, 0.1, 0);INSERT into iris (a, b, c, d, label) values (4.3, 3.0, 1.1, 0.1, 0);
+INSERT into iris (a, b, c, d, label) values (5.8, 4.0, 1.2, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.7, 4.4, 1.5, 0.4, 0);
+INSERT into iris (a, b, c, d, label) values (5.4, 3.9, 1.3, 0.4, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.5, 1.4, 0.3, 0);
+INSERT into iris (a, b, c, d, label) values (5.7, 3.8, 1.7, 0.3, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.8, 1.5, 0.3, 0);
+INSERT into iris (a, b, c, d, label) values (5.4, 3.4, 1.7, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.7, 1.5, 0.4, 0);
+INSERT into iris (a, b, c, d, label) values (4.6, 3.6, 1.0, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.3, 1.7, 0.5, 0);
+INSERT into iris (a, b, c, d, label) values (4.8, 3.4, 1.9, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.0, 1.6, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (5.0, 3.4, 1.6, 0.4, 0);INSERT into iris (a, b, c, d, label) values (5.2, 3.5, 1.5, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (5.2, 3.4, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.7, 3.2, 1.6, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (4.8, 3.1, 1.6, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.4, 3.4, 1.5, 0.4, 0);
+INSERT into iris (a, b, c, d, label) values (5.2, 4.1, 1.5, 0.1, 0);INSERT into iris (a, b, c, d, label) values (5.5, 4.2, 1.4, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (4.9, 3.1, 1.5, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.2, 1.2, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (5.5, 3.5, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.9, 3.6, 1.4, 0.1, 0);
+INSERT into iris (a, b, c, d, label) values (4.4, 3.0, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.4, 1.5, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (5.0, 3.5, 1.3, 0.3, 0);INSERT into iris (a, b, c, d, label) values (4.5, 2.3, 1.3, 0.3, 0);
+INSERT into iris (a, b, c, d, label) values (4.4, 3.2, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.5, 1.6, 0.6, 0);
+INSERT into iris (a, b, c, d, label) values (5.1, 3.8, 1.9, 0.4, 0);INSERT into iris (a, b, c, d, label) values (4.8, 3.0, 1.4, 0.3, 0);
+INSERT into iris (a, b, c, d, label) values (5.1, 3.8, 1.6, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.6, 3.2, 1.4, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (5.3, 3.7, 1.5, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.3, 1.4, 0.2, 0);
+INSERT into iris (a, b, c, d, label) values (7.0, 3.2, 4.7, 1.4, 1);INSERT into iris (a, b, c, d, label) values (6.4, 3.2, 4.5, 1.5, 1);
+INSERT into iris (a, b, c, d, label) values (6.9, 3.1, 4.9, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.5, 2.3, 4.0, 1.3, 1);
+INSERT into iris (a, b, c, d, label) values (6.5, 2.8, 4.6, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.7, 2.8, 4.5, 1.3, 1);
+INSERT into iris (a, b, c, d, label) values (6.3, 3.3, 4.7, 1.6, 1);INSERT into iris (a, b, c, d, label) values (4.9, 2.4, 3.3, 1.0, 1);
+INSERT into iris (a, b, c, d, label) values (6.6, 2.9, 4.6, 1.3, 1);INSERT into iris (a, b, c, d, label) values (5.2, 2.7, 3.9, 1.4, 1);
+INSERT into iris (a, b, c, d, label) values (5.0, 2.0, 3.5, 1.0, 1);INSERT into iris (a, b, c, d, label) values (5.9, 3.0, 4.2, 1.5, 1);
+INSERT into iris (a, b, c, d, label) values (6.0, 2.2, 4.0, 1.0, 1);INSERT into iris (a, b, c, d, label) values (6.1, 2.9, 4.7, 1.4, 1);
+INSERT into iris (a, b, c, d, label) values (5.6, 2.9, 3.6, 1.3, 1);INSERT into iris (a, b, c, d, label) values (6.7, 3.1, 4.4, 1.4, 1);
+INSERT into iris (a, b, c, d, label) values (5.6, 3.0, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 4.1, 1.0, 1);
+INSERT into iris (a, b, c, d, label) values (6.2, 2.2, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.6, 2.5, 3.9, 1.1, 1);
+INSERT into iris (a, b, c, d, label) values (5.9, 3.2, 4.8, 1.8, 1);INSERT into iris (a, b, c, d, label) values (6.1, 2.8, 4.0, 1.3, 1);
+INSERT into iris (a, b, c, d, label) values (6.3, 2.5, 4.9, 1.5, 1);INSERT into iris (a, b, c, d, label) values (6.1, 2.8, 4.7, 1.2, 1);
+INSERT into iris (a, b, c, d, label) values (6.4, 2.9, 4.3, 1.3, 1);INSERT into iris (a, b, c, d, label) values (6.6, 3.0, 4.4, 1.4, 1);
+INSERT into iris (a, b, c, d, label) values (6.8, 2.8, 4.8, 1.4, 1);INSERT into iris (a, b, c, d, label) values (6.7, 3.0, 5.0, 1.7, 1);
+INSERT into iris (a, b, c, d, label) values (6.0, 2.9, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.7, 2.6, 3.5, 1.0, 1);
+INSERT into iris (a, b, c, d, label) values (5.5, 2.4, 3.8, 1.1, 1);INSERT into iris (a, b, c, d, label) values (5.5, 2.4, 3.7, 1.0, 1);
+INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 3.9, 1.2, 1);INSERT into iris (a, b, c, d, label) values (6.0, 2.7, 5.1, 1.6, 1);
+INSERT into iris (a, b, c, d, label) values (5.4, 3.0, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (6.0, 3.4, 4.5, 1.6, 1);
+INSERT into iris (a, b, c, d, label) values (6.7, 3.1, 4.7, 1.5, 1);INSERT into iris (a, b, c, d, label) values (6.3, 2.3, 4.4, 1.3, 1);
+INSERT into iris (a, b, c, d, label) values (5.6, 3.0, 4.1, 1.3, 1);INSERT into iris (a, b, c, d, label) values (5.5, 2.5, 4.0, 1.3, 1);
+INSERT into iris (a, b, c, d, label) values (5.5, 2.6, 4.4, 1.2, 1);INSERT into iris (a, b, c, d, label) values (6.1, 3.0, 4.6, 1.4, 1);
+INSERT into iris (a, b, c, d, label) values (5.8, 2.6, 4.0, 1.2, 1);INSERT into iris (a, b, c, d, label) values (5.0, 2.3, 3.3, 1.0, 1);
+INSERT into iris (a, b, c, d, label) values (5.6, 2.7, 4.2, 1.3, 1);INSERT into iris (a, b, c, d, label) values (5.7, 3.0, 4.2, 1.2, 1);
+INSERT into iris (a, b, c, d, label) values (5.7, 2.9, 4.2, 1.3, 1);INSERT into iris (a, b, c, d, label) values (6.2, 2.9, 4.3, 1.3, 1);
+INSERT into iris (a, b, c, d, label) values (5.1, 2.5, 3.0, 1.1, 1);INSERT into iris (a, b, c, d, label) values (5.7, 2.8, 4.1, 1.3, 1);
+INSERT into iris (a, b, c, d, label) values (6.3, 3.3, 6.0, 2.5, 2);INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 5.1, 1.9, 2);
+INSERT into iris (a, b, c, d, label) values (7.1, 3.0, 5.9, 2.1, 2);INSERT into iris (a, b, c, d, label) values (6.3, 2.9, 5.6, 1.8, 2);
+INSERT into iris (a, b, c, d, label) values (6.5, 3.0, 5.8, 2.2, 2);INSERT into iris (a, b, c, d, label) values (7.6, 3.0, 6.6, 2.1, 2);
+INSERT into iris (a, b, c, d, label) values (4.9, 2.5, 4.5, 1.7, 2);INSERT into iris (a, b, c, d, label) values (7.3, 2.9, 6.3, 1.8, 2);
+INSERT into iris (a, b, c, d, label) values (6.7, 2.5, 5.8, 1.8, 2);INSERT into iris (a, b, c, d, label) values (7.2, 3.6, 6.1, 2.5, 2);
+INSERT into iris (a, b, c, d, label) values (6.5, 3.2, 5.1, 2.0, 2);INSERT into iris (a, b, c, d, label) values (6.4, 2.7, 5.3, 1.9, 2);
+INSERT into iris (a, b, c, d, label) values (6.8, 3.0, 5.5, 2.1, 2);INSERT into iris (a, b, c, d, label) values (5.7, 2.5, 5.0, 2.0, 2);
+INSERT into iris (a, b, c, d, label) values (5.8, 2.8, 5.1, 2.4, 2);INSERT into iris (a, b, c, d, label) values (6.4, 3.2, 5.3, 2.3, 2);
+INSERT into iris (a, b, c, d, label) values (6.5, 3.0, 5.5, 1.8, 2);INSERT into iris (a, b, c, d, label) values (7.7, 3.8, 6.7, 2.2, 2);
+INSERT into iris (a, b, c, d, label) values (7.7, 2.6, 6.9, 2.3, 2);INSERT into iris (a, b, c, d, label) values (6.0, 2.2, 5.0, 1.5, 2);
+INSERT into iris (a, b, c, d, label) values (6.9, 3.2, 5.7, 2.3, 2);INSERT into iris (a, b, c, d, label) values (5.6, 2.8, 4.9, 2.0, 2);
+INSERT into iris (a, b, c, d, label) values (7.7, 2.8, 6.7, 2.0, 2);INSERT into iris (a, b, c, d, label) values (6.3, 2.7, 4.9, 1.8, 2);
+INSERT into iris (a, b, c, d, label) values (6.7, 3.3, 5.7, 2.1, 2);INSERT into iris (a, b, c, d, label) values (7.2, 3.2, 6.0, 1.8, 2);
+INSERT into iris (a, b, c, d, label) values (6.2, 2.8, 4.8, 1.8, 2);INSERT into iris (a, b, c, d, label) values (6.1, 3.0, 4.9, 1.8, 2);
+INSERT into iris (a, b, c, d, label) values (6.4, 2.8, 5.6, 2.1, 2);INSERT into iris (a, b, c, d, label) values (7.2, 3.0, 5.8, 1.6, 2);
+INSERT into iris (a, b, c, d, label) values (7.4, 2.8, 6.1, 1.9, 2);INSERT into iris (a, b, c, d, label) values (7.9, 3.8, 6.4, 2.0, 2);
+INSERT into iris (a, b, c, d, label) values (6.4, 2.8, 5.6, 2.2, 2);INSERT into iris (a, b, c, d, label) values (6.3, 2.8, 5.1, 1.5, 2);
+INSERT into iris (a, b, c, d, label) values (6.1, 2.6, 5.6, 1.4, 2);INSERT into iris (a, b, c, d, label) values (7.7, 3.0, 6.1, 2.3, 2);
+INSERT into iris (a, b, c, d, label) values (6.3, 3.4, 5.6, 2.4, 2);INSERT into iris (a, b, c, d, label) values (6.4, 3.1, 5.5, 1.8, 2);
+INSERT into iris (a, b, c, d, label) values (6.0, 3.0, 4.8, 1.8, 2);INSERT into iris (a, b, c, d, label) values (6.9, 3.1, 5.4, 2.1, 2);
+INSERT into iris (a, b, c, d, label) values (6.7, 3.1, 5.6, 2.4, 2);INSERT into iris (a, b, c, d, label) values (6.9, 3.1, 5.1, 2.3, 2);
+INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 5.1, 1.9, 2);INSERT into iris (a, b, c, d, label) values (6.8, 3.2, 5.9, 2.3, 2);
+INSERT into iris (a, b, c, d, label) values (6.7, 3.3, 5.7, 2.5, 2);INSERT into iris (a, b, c, d, label) values (6.7, 3.0, 5.2, 2.3, 2);
+INSERT into iris (a, b, c, d, label) values (6.3, 2.5, 5.0, 1.9, 2);INSERT into iris (a, b, c, d, label) values (6.5, 3.0, 5.2, 2.0, 2);
+INSERT into iris (a, b, c, d, label) values (6.2, 3.4, 5.4, 2.3, 2);INSERT into iris (a, b, c, d, label) values (5.9, 3.0, 5.1, 1.8, 2);
+
+drop table if exists iris_model_xgbc;
+select MADLIB_SCHEMA.xgboost_sk_Classifier('iris', 'iris_model_xgbc', 'id', 'label', 'a,b,c,d', NULL, $${'booster': ['gbtree'], 'eta': (0.1, 0.9), 'max_depth': (5,1), 'objective': ('multi:softmax',)}$$);
+
+select id, train_timestamp, source_table, y_type, metrics, features, params from iris_model_xgbc;
+
+select assert(count(*) = 4, 'xgboost: xgboost parameter counts error') from iris_model_xgbc;
+
+select assert(metrics = 'evalution is OFF, no metrics', 'xgboost: xgboost metrics error') from iris_model_xgbc where id = 1;
+
+----------------
+drop table if exists iris_model_xgbc;
+select MADLIB_SCHEMA.xgboost_sk_Classifier('iris', 'iris_model_xgbc', 'id', 'label', 'a,b,c,d', NULL, $${'booster': ['gbtree'], 'eta': (0.1, 0.9), 'max_depth': (5,1), 'objective': ('multi:softmax',)}$$, TRUE);
+
+select id, train_timestamp, source_table, y_type, metrics, features, params from iris_model_xgbc;
+select assert(count(*) = 4, 'xgboost: xgboost parameter counts error') from iris_model_xgbc;
+select assert(metrics <> 'evalution is OFF, no metrics', 'xgboost: xgboost metrics error') from iris_model_xgbc where id = 1;
+
+-----------
+drop table if exists iris_xgbc_out;
+select MADLIB_SCHEMA.xgboost_sk_predict('iris', 'iris_model_xgbc', 'iris_xgbc_out', 'id');
+
+drop table if exists iris_model_xgbc;
+select xgboost_sk_Classifier('iris', 'iris_model_xgbc', 'id', 'label', '*', NULL, $${'booster': ('gbtree',), 'eta': [0.1, 0.9], 'max_depth': (5,1), 'objective': ['multi:softmax']}$$);
+
+SELECT assert(count(*) = 4, 'xgboost: Xgboost model output has incorrect number of rows')
+FROM (select train_timestamp, source_table, y_type, metrics, features, params, id from iris_model_xgbc) q;
+
+drop table if exists iris_xgbc_out;
+select MADLIB_SCHEMA.xgboost_sk_predict('iris', 'iris_model_xgbc', 'iris_xgbc_out', 'id');
+
+drop table if exists temp_iris;
+create table temp_iris as select t1.id, prediction, label from iris as t1, iris_xgbc_out as t2 where t1.id = t2.id order by t1.id;
+
+select assert(count(*) >135, 'xgboost: xgboost prediction incorrect')
+from (select * from temp_iris where prediction = label) q;
+
+
+----------------------------------------------------------- binaryl classification ----------------------------------------------
+
+drop table if exists xgbctable;
+create table xgbctable(id serial, "X" float, Y float, label int);
+
+insert into xgbctable("X", y, label) values (1.349880, 3.503074, 0);insert into xgbctable("X", y, label) values (3.605580, 4.285024, 0);
+insert into xgbctable("X", y, label) values (2.537560, 5.671145, 0);insert into xgbctable("X", y, label) values (2.596382, 4.219879, 0);
+insert into xgbctable("X", y, label) values (3.043375, 3.425613, 0);insert into xgbctable("X", y, label) values (2.277542, 2.532572, 0);
+insert into xgbctable("X", y, label) values (4.036647, 3.891047, 0);insert into xgbctable("X", y, label) values (3.814163, 3.737898, 0);
+insert into xgbctable("X", y, label) values (2.674975, 4.340797, 0);insert into xgbctable("X", y, label) values (1.694112, 4.533308, 0);
+insert into xgbctable("X", y, label) values (2.623954, 4.337177, 0);insert into xgbctable("X", y, label) values (3.412750, 6.994476, 0);
+insert into xgbctable("X", y, label) values (3.562286, 3.827778, 0);insert into xgbctable("X", y, label) values (3.278937, 4.116019, 0);
+insert into xgbctable("X", y, label) values (2.691412, 3.482750, 0);insert into xgbctable("X", y, label) values (1.947223, 3.734410, 0);
+insert into xgbctable("X", y, label) values (1.376026, 3.232049, 0);insert into xgbctable("X", y, label) values (1.499600, 7.429014, 0);
+insert into xgbctable("X", y, label) values (3.971720, 4.967350, 0);insert into xgbctable("X", y, label) values (2.984960, 3.780627, 0);
+insert into xgbctable("X", y, label) values (3.493389, 3.693761, 0);insert into xgbctable("X", y, label) values (3.653253, 4.156218, 0);
+insert into xgbctable("X", y, label) values (2.552306, 3.436848, 0);insert into xgbctable("X", y, label) values (3.395667, 2.731429, 0);
+insert into xgbctable("X", y, label) values (4.356895, 3.604831, 0);insert into xgbctable("X", y, label) values (2.703829, 2.192765, 0);
+insert into xgbctable("X", y, label) values (2.507021, 4.329131, 0);insert into xgbctable("X", y, label) values (5.437925, 4.798840, 0);
+insert into xgbctable("X", y, label) values (4.275079, 4.472619, 0);insert into xgbctable("X", y, label) values (3.829493, 5.539909, 0);
+insert into xgbctable("X", y, label) values (2.148925, 4.254559, 0);insert into xgbctable("X", y, label) values (2.798221, 3.916840, 0);
+insert into xgbctable("X", y, label) values (3.388155, 3.060772, 0);insert into xgbctable("X", y, label) values (2.741393, 2.502708, 0);
+insert into xgbctable("X", y, label) values (3.930997, 3.435667, 0);insert into xgbctable("X", y, label) values (2.641057, 4.070123, 0);
+insert into xgbctable("X", y, label) values (1.631679, 5.890544, 0);insert into xgbctable("X", y, label) values (3.857039, 3.486820, 0);
+insert into xgbctable("X", y, label) values (4.009723, 5.215680, 0);insert into xgbctable("X", y, label) values (4.815413, 3.742883, 0);
+insert into xgbctable("X", y, label) values (2.010829, 3.793627, 0);insert into xgbctable("X", y, label) values (3.803213, 4.966541, 0);
+insert into xgbctable("X", y, label) values (3.074533, 3.769696, 0);insert into xgbctable("X", y, label) values (3.305067, 3.941329, 0);
+insert into xgbctable("X", y, label) values (4.255057, 2.714015, 0);insert into xgbctable("X", y, label) values (1.990719, 3.748694, 0);
+insert into xgbctable("X", y, label) values (3.271186, 1.932663, 0);insert into xgbctable("X", y, label) values (5.712751, 4.925150, 0);
+insert into xgbctable("X", y, label) values (4.436121, 1.918319, 0);insert into xgbctable("X", y, label) values (1.467938, 3.670151, 0);
+insert into xgbctable("X", y, label) values (2.986207, 4.793286, 0);insert into xgbctable("X", y, label) values (2.599137, 2.807718, 0);
+insert into xgbctable("X", y, label) values (3.108875, 2.852789, 0);insert into xgbctable("X", y, label) values (3.031777, 4.673723, 0);
+insert into xgbctable("X", y, label) values (3.262225, 2.773082, 0);insert into xgbctable("X", y, label) values (2.022650, 4.607023, 0);
+insert into xgbctable("X", y, label) values (3.147877, 3.796113, 0);insert into xgbctable("X", y, label) values (3.111487, 3.129382, 0);
+insert into xgbctable("X", y, label) values (2.982505, 4.189220, 0);insert into xgbctable("X", y, label) values (2.928811, 4.102743, 0);
+insert into xgbctable("X", y, label) values (3.121620, 5.194476, 0);insert into xgbctable("X", y, label) values (3.520649, 3.870720, 0);
+insert into xgbctable("X", y, label) values (1.329847, 4.796737, 0);insert into xgbctable("X", y, label) values (4.355449, 3.361799, 0);
+insert into xgbctable("X", y, label) values (3.125785, 4.588572, 0);insert into xgbctable("X", y, label) values (2.465354, 3.974886, 0);
+insert into xgbctable("X", y, label) values (4.031982, 3.994294, 0);insert into xgbctable("X", y, label) values (2.649743, 4.096057, 0);
+insert into xgbctable("X", y, label) values (3.613181, 3.801199, 0);insert into xgbctable("X", y, label) values (6.107873, 4.124081, 0);
+insert into xgbctable("X", y, label) values (3.574605, 4.029697, 0);insert into xgbctable("X", y, label) values (2.426873, 2.782740, 0);
+insert into xgbctable("X", y, label) values (2.159410, 3.899774, 0);insert into xgbctable("X", y, label) values (2.625650, 2.297274, 0);
+insert into xgbctable("X", y, label) values (3.069545, 4.923757, 0);insert into xgbctable("X", y, label) values (2.950428, 2.858376, 0);
+insert into xgbctable("X", y, label) values (3.601556, 2.587292, 0);insert into xgbctable("X", y, label) values (3.639818, 4.304443, 0);
+insert into xgbctable("X", y, label) values (2.759995, 2.999176, 0);insert into xgbctable("X", y, label) values (1.826029, 3.564573, 0);
+insert into xgbctable("X", y, label) values (4.016062, 3.730916, 0);insert into xgbctable("X", y, label) values (2.464892, 3.112348, 0);
+insert into xgbctable("X", y, label) values (4.325916, 5.596980, 0);insert into xgbctable("X", y, label) values (2.131952, 4.396857, 0);
+insert into xgbctable("X", y, label) values (3.473718, 4.673927, 0);insert into xgbctable("X", y, label) values (3.711892, 4.908759, 0);
+insert into xgbctable("X", y, label) values (3.700971, 3.462295, 0);insert into xgbctable("X", y, label) values (3.125731, 5.828338, 0);
+insert into xgbctable("X", y, label) values (3.957516, 5.104233, 0);insert into xgbctable("X", y, label) values (4.658009, 5.078593, 0);
+insert into xgbctable("X", y, label) values (2.576429, 3.274793, 0);insert into xgbctable("X", y, label) values (3.160974, 3.466810, 0);
+insert into xgbctable("X", y, label) values (2.613970, 2.838497, 0);insert into xgbctable("X", y, label) values (2.385694, 3.749695, 0);
+insert into xgbctable("X", y, label) values (2.640589, 2.259807, 0);insert into xgbctable("X", y, label) values (4.589973, 2.776422, 0);
+insert into xgbctable("X", y, label) values (1.875313, 3.571599, 0);insert into xgbctable("X", y, label) values (3.472463, 5.251267, 0);
+insert into xgbctable("X", y, label) values (2.638684, 4.845147, 0);insert into xgbctable("X", y, label) values (3.405707, 2.662543, 0);
+
+insert into xgbctable("X", y, label) values (3.092827, -0.757907, 1);insert into xgbctable("X", y, label) values (1.982449, 0.781236, 1);
+insert into xgbctable("X", y, label) values (1.880089, 3.638690, 1);insert into xgbctable("X", y, label) values (-0.143234, -0.823097, 1);
+insert into xgbctable("X", y, label) values (1.913393, 0.717394, 1);insert into xgbctable("X", y, label) values (-0.406319, 1.495522, 1);
+insert into xgbctable("X", y, label) values (1.982796, -0.043737, 1);insert into xgbctable("X", y, label) values (2.001371, 2.005737, 1);
+insert into xgbctable("X", y, label) values (1.683994, 2.722670, 1);insert into xgbctable("X", y, label) values (1.201483, -0.544765, 1);
+insert into xgbctable("X", y, label) values (1.054981, 0.857019, 1);insert into xgbctable("X", y, label) values (-1.137237, 0.769533, 1);
+insert into xgbctable("X", y, label) values (0.526662, -0.076017, 1);insert into xgbctable("X", y, label) values (0.431452, 1.018968, 1);
+insert into xgbctable("X", y, label) values (0.911789, -1.003597, 1);insert into xgbctable("X", y, label) values (0.271256, 0.749327, 1);
+insert into xgbctable("X", y, label) values (-0.141832, -1.478830, 1);insert into xgbctable("X", y, label) values (0.508604, 0.648365, 1);
+insert into xgbctable("X", y, label) values (1.903264, 0.034425, 1);insert into xgbctable("X", y, label) values (0.788896, 0.615347, 1);
+insert into xgbctable("X", y, label) values (3.350388, -0.238897, 1);insert into xgbctable("X", y, label) values (-0.632393, 0.605239, 1);
+insert into xgbctable("X", y, label) values (1.170017, 0.536877, 1);insert into xgbctable("X", y, label) values (1.381837, 0.453970, 1);
+insert into xgbctable("X", y, label) values (0.238530, 0.234563, 1);insert into xgbctable("X", y, label) values (-0.619639, -0.739849, 1);
+insert into xgbctable("X", y, label) values (1.782465, 2.802568, 1);insert into xgbctable("X", y, label) values (-1.690702, 0.989325, 1);
+insert into xgbctable("X", y, label) values (0.109266, 2.755468, 1);insert into xgbctable("X", y, label) values (1.670044, 1.392086, 1);
+insert into xgbctable("X", y, label) values (2.461095, -0.625544, 1);insert into xgbctable("X", y, label) values (1.580424, 1.225731, 1);
+insert into xgbctable("X", y, label) values (0.949291, 0.709720, 1);insert into xgbctable("X", y, label) values (1.919173, 0.810358, 1);
+insert into xgbctable("X", y, label) values (1.086786, 2.770693, 1);insert into xgbctable("X", y, label) values (0.955707, 1.763802, 1);
+insert into xgbctable("X", y, label) values (1.689414, 1.062906, 1);insert into xgbctable("X", y, label) values (1.438960, 1.282794, 1);
+insert into xgbctable("X", y, label) values (0.499648, 1.425253, 1);insert into xgbctable("X", y, label) values (0.712802, -0.080972, 1);
+insert into xgbctable("X", y, label) values (1.501847, 1.318616, 1);insert into xgbctable("X", y, label) values (-0.718307, 0.644610, 1);
+insert into xgbctable("X", y, label) values (-0.259394, 1.004450, 1);insert into xgbctable("X", y, label) values (1.822353, 1.461093, 1);
+insert into xgbctable("X", y, label) values (0.065569, 2.134341, 1);insert into xgbctable("X", y, label) values (1.137744, 1.431270, 1);
+insert into xgbctable("X", y, label) values (1.613366, -0.399205, 1);insert into xgbctable("X", y, label) values (2.780135, 0.523116, 1);
+insert into xgbctable("X", y, label) values (1.641254, 0.189915, 1);insert into xgbctable("X", y, label) values (0.224779, 0.320547, 1);
+insert into xgbctable("X", y, label) values (1.509881, 0.488949, 1);insert into xgbctable("X", y, label) values (2.742344, 1.078382, 1);
+insert into xgbctable("X", y, label) values (0.015037, 0.906546, 1);insert into xgbctable("X", y, label) values (0.790595, 0.721075, 1);
+insert into xgbctable("X", y, label) values (0.296409, 0.922898, 1);insert into xgbctable("X", y, label) values (1.116124, 2.510543, 1);
+insert into xgbctable("X", y, label) values (1.104642, 1.043124, 1);insert into xgbctable("X", y, label) values (1.261002, 1.275877, 1);
+insert into xgbctable("X", y, label) values (0.995230, 1.131128, 1);insert into xgbctable("X", y, label) values (0.500961, 1.053070, 1);
+insert into xgbctable("X", y, label) values (0.753316, 1.587285, 1);insert into xgbctable("X", y, label) values (1.415908, 1.080478, 1);
+insert into xgbctable("X", y, label) values (1.391515, 0.915593, 1);insert into xgbctable("X", y, label) values (1.309082, 0.667459, 1);
+insert into xgbctable("X", y, label) values (0.622153, 2.885750, 1);insert into xgbctable("X", y, label) values (2.499712, 1.917895, 1);
+insert into xgbctable("X", y, label) values (0.754728, 1.182577, 1);insert into xgbctable("X", y, label) values (1.653910, -0.448352, 1);
+insert into xgbctable("X", y, label) values (2.263763, -0.022151, 1);insert into xgbctable("X", y, label) values (-0.289203, 2.267931, 1);
+insert into xgbctable("X", y, label) values (2.117514, 0.852814, 1);insert into xgbctable("X", y, label) values (0.682405, 0.043468, 1);
+insert into xgbctable("X", y, label) values (1.304387, 1.649660, 1);insert into xgbctable("X", y, label) values (2.775033, 0.608014, 1);
+insert into xgbctable("X", y, label) values (0.695554, 1.631088, 1);insert into xgbctable("X", y, label) values (1.736803, 0.258182, 1);
+insert into xgbctable("X", y, label) values (1.369698, 1.281675, 1);insert into xgbctable("X", y, label) values (-0.461257, 2.632165, 1);
+insert into xgbctable("X", y, label) values (1.672385, 0.965322, 1);insert into xgbctable("X", y, label) values (0.527026, 0.849661, 1);
+insert into xgbctable("X", y, label) values (1.583187, 0.823039, 1);insert into xgbctable("X", y, label) values (1.217844, 1.465343, 1);
+insert into xgbctable("X", y, label) values (-0.190087, -0.190168, 1);insert into xgbctable("X", y, label) values (0.939329, -0.837490, 1);
+insert into xgbctable("X", y, label) values (0.421842, -1.023412, 1);insert into xgbctable("X", y, label) values (1.015774, 0.422640, 1);
+insert into xgbctable("X", y, label) values (0.036181, -1.399000, 1);insert into xgbctable("X", y, label) values (1.001890, 1.183682, 1);
+insert into xgbctable("X", y, label) values (3.219715, 1.811829, 1);insert into xgbctable("X", y, label) values (2.270468, 1.717835, 1);
+insert into xgbctable("X", y, label) values (0.354725, 0.918786, 1);insert into xgbctable("X", y, label) values (3.065704, 0.358424, 1);
+insert into xgbctable("X", y, label) values (-0.609859, 0.152997, 1);insert into xgbctable("X", y, label) values (0.476704, 0.071017, 1);
+insert into xgbctable("X", y, label) values (0.494553, 3.201871, 1);insert into xgbctable("X", y, label) values (3.430162, -0.834660, 1);
+insert into xgbctable("X", y, label) values (2.542903, 1.831189, 1);insert into xgbctable("X", y, label) values (1.973389, 0.079485, 1);
+insert into xgbctable("X", y, label) values (1.309454, 0.648461, 1);insert into xgbctable("X", y, label) values (0.841486, -0.177773, 1);
+
+drop table if exists xgbctable_model;
+select MADLIB_SCHEMA.xgboost_sk_Classifier('xgbctable', 'xgbctable_model', 'id', 'label', '"X", Y', NULL, $${'booster': ('gbtree',), 'eta': (0.1), 'max_depth': 5, 'objective': ('binary:logistic',)}$$, TRUE);
+
+select id, train_timestamp, source_table, y_type, metrics, features, params from xgbctable_model;
+select assert(count(*) = 1, 'xgboost: xgboost parameter counts error') from xgbctable_model;
+
+drop table if exists xgbctable_model;
+select MADLIB_SCHEMA.xgboost_sk_Classifier('xgbctable', 'xgbctable_model', 'id', 'label', '*', 'id, label', $${'booster': ['gbtree'], 'eta': 0.1, 'max_depth': (5), 'objective': ['binary:logistic']}$$);
+select assert(metrics = 'evalution is OFF, no metrics', 'xgboost: evalution error') from xgbctable_model;
+
+
+drop table if exists xgbctable_output;
+select MADLIB_SCHEMA.xgboost_sk_predict('xgbctable', 'xgbctable_model', 'xgbctable_output', 'id', TRUE);
+
+drop table if exists xgbctable_output_summary;
+create table xgbctable_output_summary as select t1.id, prediction, label from xgbctable_output as t1, xgbctable as t2 where t1.id = t2.id order by t1.id;
+
+drop table if exists xgbctable_aur;
+select MADLIB_SCHEMA.area_under_roc('xgbctable_output_summary', 'xgbctable_aur', 'prediction', 'label');
+select * from xgbctable_aur;
+select assert(area_under_roc > 0.9, 'xgbctable AUR error') from xgbctable_aur;
+
+drop table if exists xgbctable_cm;
+select MADLIB_SCHEMA.confusion_matrix('xgbctable_output_summary', 'xgbctable_cm', 'prediction', 'label');
+select * from xgbctable_cm;
diff --git a/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_regress.sql_in b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_regress.sql_in
new file mode 100644
index 0000000..99e216a
--- /dev/null
+++ b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_regress.sql_in
@@ -0,0 +1,554 @@
+/*
+ * Test case for sklearn-api xgboost.
+ */
+
+m4_include(`SQLCommon.m4')
+
+select xgboost_sk_regressor('usage');
+select xgboost_sk_regressor('help');
+select xgboost_sk_regressor('?');
+select xgboost_sk_regressor('');
+
+DROP TABLE IF EXISTS boston;
+CREATE TABLE boston (
+id integer NOT NULL, crim double precision, zn float8, Indus double precision,
+chas double precision, nox float, rm float4, "Age" int,
+dis DECIMAL(4), rad double precision, tax double precision, ptratio float,
+b double precision, LstaT double precision, target float);
+
+INSERT INTO boston VALUES (2, 0.027310000000000001, 0, 7.0700000000000003, 0, 0.46899999999999997, 6.4210000000000003, 78.900000000000006, 4.9671000000000003, 2, 242, 17.800000000000001, 396.89999999999998, 9.1400000000000006, 21.600000000000001);
+INSERT INTO boston VALUES (6, 0.029850000000000002, 0, 2.1800000000000002, 0, 0.45800000000000002, 6.4299999999999997, 58.700000000000003, 6.0621999999999998, 3, 222, 18.699999999999999, 394.12, 5.21, 28.699999999999999);
+INSERT INTO boston VALUES (10, 0.17004, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.0039999999999996, 85.900000000000006, 6.5921000000000003, 5, 311, 15.199999999999999, 386.70999999999998, 17.100000000000001, 18.899999999999999);
+INSERT INTO boston VALUES (14, 0.62975999999999999, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9489999999999998, 61.799999999999997, 4.7074999999999996, 4, 307, 21, 396.89999999999998, 8.2599999999999998, 20.399999999999999);
+INSERT INTO boston VALUES (18, 0.78420000000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9900000000000002, 81.700000000000003, 4.2579000000000002, 4, 307, 21, 386.75, 14.67, 17.5);
+INSERT INTO boston VALUES (22, 0.85204000000000002, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9649999999999999, 89.200000000000003, 4.0122999999999998, 4, 307, 21, 392.52999999999997, 13.83, 19.600000000000001);
+INSERT INTO boston VALUES (26, 0.84053999999999995, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.5990000000000002, 85.700000000000003, 4.4546000000000001, 4, 307, 21, 303.42000000000002, 16.510000000000002, 13.9);
+INSERT INTO boston VALUES (30, 1.0024500000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.6740000000000004, 87.299999999999997, 4.2389999999999999, 4, 307, 21, 380.23000000000002, 11.98, 21);
+INSERT INTO boston VALUES (34, 1.1517200000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.7009999999999996, 95, 3.7871999999999999, 4, 307, 21, 358.76999999999998, 18.350000000000001, 13.1);
+INSERT INTO boston VALUES (38, 0.080140000000000003, 0, 5.96, 0, 0.499, 5.8499999999999996, 41.5, 3.9342000000000001, 5, 279, 19.199999999999999, 396.89999999999998, 8.7699999999999996, 21);
+INSERT INTO boston VALUES (42, 0.12744, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.7699999999999996, 2.8999999999999999, 5.7209000000000003, 3, 233, 17.899999999999999, 385.41000000000003, 4.8399999999999999, 26.600000000000001);
+INSERT INTO boston VALUES (46, 0.17141999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.6820000000000004, 33.799999999999997, 5.1003999999999996, 3, 233, 17.899999999999999, 396.89999999999998, 10.210000000000001, 19.300000000000001);
+INSERT INTO boston VALUES (50, 0.21976999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.6020000000000003, 62, 6.0876999999999999, 3, 233, 17.899999999999999, 396.89999999999998, 16.199999999999999, 19.399999999999999);
+INSERT INTO boston VALUES (54, 0.04981, 21, 5.6399999999999997, 0, 0.439, 5.9980000000000002, 21.399999999999999, 6.8147000000000002, 4, 243, 16.800000000000001, 396.89999999999998, 8.4299999999999997, 23.399999999999999);
+INSERT INTO boston VALUES (58, 0.014319999999999999, 100, 1.3200000000000001, 0, 0.41099999999999998, 6.8159999999999998, 40.5, 8.3247999999999998, 5, 256, 15.1, 392.89999999999998, 3.9500000000000002, 31.600000000000001);
+INSERT INTO boston VALUES (62, 0.17171, 25, 5.1299999999999999, 0, 0.45300000000000001, 5.9660000000000002, 93.400000000000006, 6.8185000000000002, 8, 284, 19.699999999999999, 378.07999999999998, 14.44, 16);
+INSERT INTO boston VALUES (66, 0.035839999999999997, 80, 3.3700000000000001, 0, 0.39800000000000002, 6.29, 17.800000000000001, 6.6115000000000004, 4, 337, 16.100000000000001, 396.89999999999998, 4.6699999999999999, 23.5);
+INSERT INTO boston VALUES (70, 0.12816, 12.5, 6.0700000000000003, 0, 0.40899999999999997, 5.8849999999999998, 33, 6.4980000000000002, 4, 345, 18.899999999999999, 396.89999999999998, 8.7899999999999991, 20.899999999999999);
+INSERT INTO boston VALUES (74, 0.19539000000000001, 0, 10.81, 0, 0.41299999999999998, 6.2450000000000001, 6.2000000000000002, 5.2873000000000001, 4, 305, 19.199999999999999, 377.17000000000002, 7.54, 23.399999999999999);
+INSERT INTO boston VALUES (78, 0.087069999999999995, 0, 12.83, 0, 0.437, 6.1399999999999997, 45.799999999999997, 4.0904999999999996, 5, 398, 18.699999999999999, 386.95999999999998, 10.27, 20.800000000000001);
+INSERT INTO boston VALUES (82, 0.04462, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.6189999999999998, 70.400000000000006, 5.4006999999999996, 4, 281, 19, 395.63, 7.2199999999999998, 23.899999999999999);
+INSERT INTO boston VALUES (86, 0.057349999999999998, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.6299999999999999, 56.100000000000001, 4.4377000000000004, 3, 247, 18.5, 392.30000000000001, 6.5300000000000002, 26.600000000000001);
+INSERT INTO boston VALUES (90, 0.053019999999999998, 0, 3.4100000000000001, 0, 0.48899999999999999, 7.0789999999999997, 63.100000000000001, 3.4144999999999999, 2, 270, 17.800000000000001, 396.06, 5.7000000000000002, 28.699999999999999);
+INSERT INTO boston VALUES (94, 0.028750000000000001, 28, 15.039999999999999, 0, 0.46400000000000002, 6.2110000000000003, 28.899999999999999, 3.6659000000000002, 4, 270, 18.199999999999999, 396.32999999999998, 6.21, 25);
+INSERT INTO boston VALUES (98, 0.12083000000000001, 0, 2.8900000000000001, 0, 0.44500000000000001, 8.0690000000000008, 76, 3.4952000000000001, 2, 276, 18, 396.89999999999998, 4.21, 38.700000000000003);
+INSERT INTO boston VALUES (102, 0.11432, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.7809999999999997, 71.299999999999997, 2.8561000000000001, 5, 384, 20.899999999999999, 395.57999999999998, 7.6699999999999999, 26.5);
+INSERT INTO boston VALUES (106, 0.13261999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 5.851, 96.700000000000003, 2.1069, 5, 384, 20.899999999999999, 394.05000000000001, 16.469999999999999, 19.5);
+INSERT INTO boston VALUES (110, 0.26362999999999998, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.2290000000000001, 91.200000000000003, 2.5451000000000001, 5, 384, 20.899999999999999, 391.23000000000002, 15.550000000000001, 19.399999999999999);
+INSERT INTO boston VALUES (114, 0.22212000000000001, 0, 10.01, 0, 0.54700000000000004, 6.0919999999999996, 95.400000000000006, 2.548, 6, 432, 17.800000000000001, 396.89999999999998, 17.09, 18.699999999999999);
+INSERT INTO boston VALUES (118, 0.15098, 0, 10.01, 0, 0.54700000000000004, 6.0209999999999999, 82.599999999999994, 2.7473999999999998, 6, 432, 17.800000000000001, 394.50999999999999, 10.300000000000001, 19.199999999999999);
+INSERT INTO boston VALUES (122, 0.071650000000000005, 0, 25.649999999999999, 0, 0.58099999999999996, 6.0039999999999996, 84.099999999999994, 2.1974, 2, 188, 19.100000000000001, 377.67000000000002, 14.27, 20.300000000000001);
+INSERT INTO boston VALUES (126, 0.16902, 0, 25.649999999999999, 0, 0.58099999999999996, 5.9859999999999998, 88.400000000000006, 1.9928999999999999, 2, 188, 19.100000000000001, 385.01999999999998, 14.81, 21.399999999999999);
+INSERT INTO boston VALUES (130, 0.88124999999999998, 0, 21.890000000000001, 0, 0.624, 5.6369999999999996, 94.700000000000003, 1.9799, 4, 437, 21.199999999999999, 396.89999999999998, 18.34, 14.300000000000001);
+INSERT INTO boston VALUES (134, 0.32982, 0, 21.890000000000001, 0, 0.624, 5.8220000000000001, 95.400000000000006, 2.4699, 4, 437, 21.199999999999999, 388.69, 15.029999999999999, 18.399999999999999);
+INSERT INTO boston VALUES (138, 0.35232999999999998, 0, 21.890000000000001, 0, 0.624, 6.4539999999999997, 98.400000000000006, 1.8498000000000001, 4, 437, 21.199999999999999, 394.07999999999998, 14.59, 17.100000000000001);
+INSERT INTO boston VALUES (142, 1.6286400000000001, 0, 21.890000000000001, 0, 0.624, 5.0190000000000001, 100, 1.4394, 4, 437, 21.199999999999999, 396.89999999999998, 34.409999999999997, 14.4);
+INSERT INTO boston VALUES (146, 2.37934, 0, 19.579999999999998, 0, 0.871, 6.1299999999999999, 100, 1.4191, 5, 403, 14.699999999999999, 172.91, 27.800000000000001, 13.800000000000001);
+INSERT INTO boston VALUES (150, 2.7339699999999998, 0, 19.579999999999998, 0, 0.871, 5.5970000000000004, 94.900000000000006, 1.5257000000000001, 5, 403, 14.699999999999999, 351.85000000000002, 21.449999999999999, 15.4);
+INSERT INTO boston VALUES (154, 2.1491799999999999, 0, 19.579999999999998, 0, 0.871, 5.7089999999999996, 98.5, 1.6232, 5, 403, 14.699999999999999, 261.94999999999999, 15.789999999999999, 19.399999999999999);
+INSERT INTO boston VALUES (158, 1.2235799999999999, 0, 19.579999999999998, 0, 0.60499999999999998, 6.9429999999999996, 97.400000000000006, 1.8773, 5, 403, 14.699999999999999, 363.43000000000001, 4.5899999999999999, 41.299999999999997);
+INSERT INTO boston VALUES (162, 1.46336, 0, 19.579999999999998, 0, 0.60499999999999998, 7.4889999999999999, 90.799999999999997, 1.9709000000000001, 5, 403, 14.699999999999999, 374.43000000000001, 1.73, 50);
+INSERT INTO boston VALUES (166, 2.9239999999999999, 0, 19.579999999999998, 0, 0.60499999999999998, 6.101, 93, 2.2833999999999999, 5, 403, 14.699999999999999, 240.16, 9.8100000000000005, 25);
+INSERT INTO boston VALUES (170, 2.4495300000000002, 0, 19.579999999999998, 0, 0.60499999999999998, 6.4020000000000001, 95.200000000000003, 2.2625000000000002, 5, 403, 14.699999999999999, 330.04000000000002, 11.32, 22.300000000000001);
+INSERT INTO boston VALUES (174, 0.09178, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.4160000000000004, 84.099999999999994, 2.6463000000000001, 5, 296, 16.600000000000001, 395.5, 9.0399999999999991, 23.600000000000001);
+INSERT INTO boston VALUES (178, 0.05425, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.3150000000000004, 73.400000000000006, 3.3174999999999999, 5, 296, 16.600000000000001, 395.60000000000002, 6.29, 24.600000000000001);
+INSERT INTO boston VALUES (182, 0.068879999999999997, 0, 2.46, 0, 0.48799999999999999, 6.1440000000000001, 62.200000000000003, 2.5979000000000001, 3, 193, 17.800000000000001, 396.89999999999998, 9.4499999999999993, 36.200000000000003);
+INSERT INTO boston VALUES (186, 0.060470000000000003, 0, 2.46, 0, 0.48799999999999999, 6.1529999999999996, 68.799999999999997, 3.2797000000000001, 3, 193, 17.800000000000001, 387.11000000000001, 13.15, 29.600000000000001);
+INSERT INTO boston VALUES (190, 0.083699999999999997, 45, 3.4399999999999999, 0, 0.437, 7.1849999999999996, 38.899999999999999, 4.5667, 5, 398, 15.199999999999999, 396.89999999999998, 5.3899999999999997, 34.899999999999999);
+INSERT INTO boston VALUES (194, 0.021870000000000001, 60, 2.9300000000000002, 0, 0.40100000000000002, 6.7999999999999998, 9.9000000000000004, 6.2195999999999998, 1, 265, 15.6, 393.37, 5.0300000000000002, 31.100000000000001);
+INSERT INTO boston VALUES (198, 0.04666, 80, 1.52, 0, 0.40400000000000003, 7.1070000000000002, 36.600000000000001, 7.3090000000000002, 2, 329, 12.6, 354.31, 8.6099999999999994, 30.300000000000001);
+INSERT INTO boston VALUES (202, 0.034450000000000001, 82.5, 2.0299999999999998, 0, 0.41499999999999998, 6.1619999999999999, 38.399999999999999, 6.2699999999999996, 2, 348, 14.699999999999999, 393.76999999999998, 7.4299999999999997, 24.100000000000001);
+INSERT INTO boston VALUES (206, 0.13642000000000001, 0, 10.59, 0, 0.48899999999999999, 5.891, 22.300000000000001, 3.9453999999999998, 4, 277, 18.600000000000001, 396.89999999999998, 10.869999999999999, 22.600000000000001);
+INSERT INTO boston VALUES (210, 0.43570999999999999, 0, 10.59, 1, 0.48899999999999999, 5.3440000000000003, 100, 3.875, 4, 277, 18.600000000000001, 396.89999999999998, 23.09, 20);
+INSERT INTO boston VALUES (214, 0.14052000000000001, 0, 10.59, 0, 0.48899999999999999, 6.375, 32.299999999999997, 3.9453999999999998, 4, 277, 18.600000000000001, 385.81, 9.3800000000000008, 28.100000000000001);
+INSERT INTO boston VALUES (218, 0.070129999999999998, 0, 13.890000000000001, 0, 0.55000000000000004, 6.6420000000000003, 85.099999999999994, 3.4211, 5, 276, 16.399999999999999, 392.77999999999997, 9.6899999999999995, 28.699999999999999);
+INSERT INTO boston VALUES (222, 0.40771000000000002, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.1639999999999997, 91.299999999999997, 3.048, 8, 307, 17.399999999999999, 395.24000000000001, 21.460000000000001, 21.699999999999999);
+INSERT INTO boston VALUES (226, 0.52693000000000001, 0, 6.2000000000000002, 0, 0.504, 8.7249999999999996, 83, 2.8944000000000001, 8, 307, 17.399999999999999, 382, 4.6299999999999999, 50);
+INSERT INTO boston VALUES (230, 0.44178000000000001, 0, 6.2000000000000002, 0, 0.504, 6.5519999999999996, 21.399999999999999, 3.3751000000000002, 8, 307, 17.399999999999999, 380.33999999999997, 3.7599999999999998, 31.5);
+INSERT INTO boston VALUES (1, 0.0063200000000000001, 18, 2.3100000000000001, 0, 0.53800000000000003, 6.5750000000000002, 65.200000000000003, 4.0899999999999999, 1, 296, 15.300000000000001, 396.89999999999998, 4.9800000000000004, 24);
+INSERT INTO boston VALUES (5, 0.06905, 0, 2.1800000000000002, 0, 0.45800000000000002, 7.1470000000000002, 54.200000000000003, 6.0621999999999998, 3, 222, 18.699999999999999, 396.89999999999998, 5.3300000000000001, 36.200000000000003);
+INSERT INTO boston VALUES (9, 0.21124000000000001, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 5.6310000000000002, 100, 6.0820999999999996, 5, 311, 15.199999999999999, 386.63, 29.93, 16.5);
+INSERT INTO boston VALUES (13, 0.093780000000000002, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 5.8890000000000002, 39, 5.4508999999999999, 5, 311, 15.199999999999999, 390.5, 15.710000000000001, 21.699999999999999);
+INSERT INTO boston VALUES (17, 1.05393, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9349999999999996, 29.300000000000001, 4.4985999999999997, 4, 307, 21, 386.85000000000002, 6.5800000000000001, 23.100000000000001);
+INSERT INTO boston VALUES (21, 1.25179, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.5700000000000003, 98.099999999999994, 3.7978999999999998, 4, 307, 21, 376.56999999999999, 21.02, 13.6);
+INSERT INTO boston VALUES (25, 0.75026000000000004, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9240000000000004, 94.099999999999994, 4.3996000000000004, 4, 307, 21, 394.32999999999998, 16.300000000000001, 15.6);
+INSERT INTO boston VALUES (29, 0.77298999999999995, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.4950000000000001, 94.400000000000006, 4.4546999999999999, 4, 307, 21, 387.94, 12.800000000000001, 18.399999999999999);
+INSERT INTO boston VALUES (33, 1.3879900000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9500000000000002, 82, 3.9900000000000002, 4, 307, 21, 232.59999999999999, 27.710000000000001, 13.199999999999999);
+INSERT INTO boston VALUES (37, 0.097439999999999999, 0, 5.96, 0, 0.499, 5.8410000000000002, 61.399999999999999, 3.3778999999999999, 5, 279, 19.199999999999999, 377.56, 11.41, 20);
+INSERT INTO boston VALUES (41, 0.033590000000000002, 75, 2.9500000000000002, 0, 0.42799999999999999, 7.024, 15.800000000000001, 5.4010999999999996, 3, 252, 18.300000000000001, 395.62, 1.98, 34.899999999999999);
+INSERT INTO boston VALUES (45, 0.12268999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.069, 40, 5.7209000000000003, 3, 233, 17.899999999999999, 389.38999999999999, 9.5500000000000007, 21.199999999999999);
+INSERT INTO boston VALUES (49, 0.25386999999999998, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.399, 95.299999999999997, 5.8700000000000001, 3, 233, 17.899999999999999, 396.89999999999998, 30.809999999999999, 14.4);
+INSERT INTO boston VALUES (53, 0.053600000000000002, 21, 5.6399999999999997, 0, 0.439, 6.5110000000000001, 21.100000000000001, 6.8147000000000002, 4, 243, 16.800000000000001, 396.89999999999998, 5.2800000000000002, 25);
+INSERT INTO boston VALUES (57, 0.020549999999999999, 85, 0.73999999999999999, 0, 0.40999999999999998, 6.383, 35.700000000000003, 9.1875999999999998, 2, 313, 17.300000000000001, 396.89999999999998, 5.7699999999999996, 24.699999999999999);
+INSERT INTO boston VALUES (61, 0.14932000000000001, 25, 5.1299999999999999, 0, 0.45300000000000001, 5.7409999999999997, 66.200000000000003, 7.2253999999999996, 8, 284, 19.699999999999999, 395.11000000000001, 13.15, 18.699999999999999);
+INSERT INTO boston VALUES (65, 0.01951, 17.5, 1.3799999999999999, 0, 0.41610000000000003, 7.1040000000000001, 59.5, 9.2228999999999992, 3, 216, 18.600000000000001, 393.24000000000001, 8.0500000000000007, 33);
+INSERT INTO boston VALUES (69, 0.13553999999999999, 12.5, 6.0700000000000003, 0, 0.40899999999999997, 5.5940000000000003, 36.799999999999997, 6.4980000000000002, 4, 345, 18.899999999999999, 396.89999999999998, 13.09, 17.399999999999999);
+INSERT INTO boston VALUES (73, 0.091639999999999999, 0, 10.81, 0, 0.41299999999999998, 6.0650000000000004, 7.7999999999999998, 5.2873000000000001, 4, 305, 19.199999999999999, 390.91000000000003, 5.5199999999999996, 22.800000000000001);
+INSERT INTO boston VALUES (77, 0.10153, 0, 12.83, 0, 0.437, 6.2789999999999999, 74.5, 4.0522, 5, 398, 18.699999999999999, 373.66000000000003, 11.970000000000001, 20);
+INSERT INTO boston VALUES (81, 0.04113, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.7270000000000003, 33.5, 5.4006999999999996, 4, 281, 19, 396.89999999999998, 5.29, 28);
+INSERT INTO boston VALUES (85, 0.050590000000000003, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.3890000000000002, 48, 4.7793999999999999, 3, 247, 18.5, 396.89999999999998, 9.6199999999999992, 23.899999999999999);
+INSERT INTO boston VALUES (89, 0.056599999999999998, 0, 3.4100000000000001, 0, 0.48899999999999999, 7.0069999999999997, 86.299999999999997, 3.4217, 2, 270, 17.800000000000001, 396.89999999999998, 5.5, 23.600000000000001);
+INSERT INTO boston VALUES (93, 0.042029999999999998, 28, 15.039999999999999, 0, 0.46400000000000002, 6.4420000000000002, 53.600000000000001, 3.6659000000000002, 4, 270, 18.199999999999999, 395.00999999999999, 8.1600000000000001, 22.899999999999999);
+INSERT INTO boston VALUES (97, 0.11504, 0, 2.8900000000000001, 0, 0.44500000000000001, 6.1630000000000003, 69.599999999999994, 3.4952000000000001, 2, 276, 18, 391.82999999999998, 11.34, 21.399999999999999);
+INSERT INTO boston VALUES (101, 0.14865999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.7270000000000003, 79.900000000000006, 2.7778, 5, 384, 20.899999999999999, 394.75999999999999, 9.4199999999999999, 27.5);
+INSERT INTO boston VALUES (105, 0.1396, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1669999999999998, 90, 2.4209999999999998, 5, 384, 20.899999999999999, 392.69, 12.33, 20.100000000000001);
+INSERT INTO boston VALUES (109, 0.12801999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.4740000000000002, 97.099999999999994, 2.4329000000000001, 5, 384, 20.899999999999999, 395.24000000000001, 12.27, 19.800000000000001);
+INSERT INTO boston VALUES (113, 0.12329, 0, 10.01, 0, 0.54700000000000004, 5.9130000000000003, 92.900000000000006, 2.3534000000000002, 6, 432, 17.800000000000001, 394.94999999999999, 16.210000000000001, 18.800000000000001);
+INSERT INTO boston VALUES (117, 0.13158, 0, 10.01, 0, 0.54700000000000004, 6.1760000000000002, 72.5, 2.7301000000000002, 6, 432, 17.800000000000001, 393.30000000000001, 12.039999999999999, 21.199999999999999);
+INSERT INTO boston VALUES (121, 0.068989999999999996, 0, 25.649999999999999, 0, 0.58099999999999996, 5.8700000000000001, 69.700000000000003, 2.2576999999999998, 2, 188, 19.100000000000001, 389.14999999999998, 14.369999999999999, 22);
+INSERT INTO boston VALUES (125, 0.098489999999999994, 0, 25.649999999999999, 0, 0.58099999999999996, 5.8789999999999996, 95.799999999999997, 2.0063, 2, 188, 19.100000000000001, 379.38, 17.579999999999998, 18.800000000000001);
+INSERT INTO boston VALUES (129, 0.32543, 0, 21.890000000000001, 0, 0.624, 6.431, 98.799999999999997, 1.8125, 4, 437, 21.199999999999999, 396.89999999999998, 15.390000000000001, 18);
+INSERT INTO boston VALUES (133, 0.59004999999999996, 0, 21.890000000000001, 0, 0.624, 6.3719999999999999, 97.900000000000006, 2.3273999999999999, 4, 437, 21.199999999999999, 385.75999999999999, 11.119999999999999, 23);
+INSERT INTO boston VALUES (137, 0.32263999999999998, 0, 21.890000000000001, 0, 0.624, 5.9420000000000002, 93.5, 1.9669000000000001, 4, 437, 21.199999999999999, 378.25, 16.899999999999999, 17.399999999999999);
+INSERT INTO boston VALUES (141, 0.29089999999999999, 0, 21.890000000000001, 0, 0.624, 6.1740000000000004, 93.599999999999994, 1.6119000000000001, 4, 437, 21.199999999999999, 388.07999999999998, 24.16, 14);
+INSERT INTO boston VALUES (145, 2.7797399999999999, 0, 19.579999999999998, 0, 0.871, 4.9029999999999996, 97.799999999999997, 1.3459000000000001, 5, 403, 14.699999999999999, 396.89999999999998, 29.289999999999999, 11.800000000000001);
+INSERT INTO boston VALUES (149, 2.3309899999999999, 0, 19.579999999999998, 0, 0.871, 5.1859999999999999, 93.799999999999997, 1.5296000000000001, 5, 403, 14.699999999999999, 356.99000000000001, 28.32, 17.800000000000001);
+INSERT INTO boston VALUES (153, 1.1265799999999999, 0, 19.579999999999998, 1, 0.871, 5.0119999999999996, 88, 1.6102000000000001, 5, 403, 14.699999999999999, 343.27999999999997, 12.119999999999999, 15.300000000000001);
+INSERT INTO boston VALUES (157, 2.4466800000000002, 0, 19.579999999999998, 0, 0.871, 5.2720000000000002, 94, 1.7363999999999999, 5, 403, 14.699999999999999, 88.629999999999995, 16.140000000000001, 13.1);
+INSERT INTO boston VALUES (161, 1.27346, 0, 19.579999999999998, 1, 0.60499999999999998, 6.25, 92.599999999999994, 1.7984, 5, 403, 14.699999999999999, 338.92000000000002, 5.5, 27);
+INSERT INTO boston VALUES (165, 2.2423600000000001, 0, 19.579999999999998, 0, 0.60499999999999998, 5.8540000000000001, 91.799999999999997, 2.4220000000000002, 5, 403, 14.699999999999999, 395.11000000000001, 11.640000000000001, 22.699999999999999);
+INSERT INTO boston VALUES (169, 2.3003999999999998, 0, 19.579999999999998, 0, 0.60499999999999998, 6.319, 96.099999999999994, 2.1000000000000001, 5, 403, 14.699999999999999, 297.08999999999997, 11.1, 23.800000000000001);
+INSERT INTO boston VALUES (173, 0.13914000000000001, 0, 4.0499999999999998, 0, 0.51000000000000001, 5.5720000000000001, 88.5, 2.5960999999999999, 5, 296, 16.600000000000001, 396.89999999999998, 14.69, 23.100000000000001);
+INSERT INTO boston VALUES (177, 0.070220000000000005, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.0199999999999996, 47.200000000000003, 3.5548999999999999, 5, 296, 16.600000000000001, 393.23000000000002, 10.109999999999999, 23.199999999999999);
+INSERT INTO boston VALUES (181, 0.065879999999999994, 0, 2.46, 0, 0.48799999999999999, 7.7649999999999997, 83.299999999999997, 2.7410000000000001, 3, 193, 17.800000000000001, 395.56, 7.5599999999999996, 39.799999999999997);
+INSERT INTO boston VALUES (185, 0.083080000000000001, 0, 2.46, 0, 0.48799999999999999, 5.6040000000000001, 89.799999999999997, 2.9878999999999998, 3, 193, 17.800000000000001, 391, 13.98, 26.399999999999999);
+INSERT INTO boston VALUES (189, 0.12579000000000001, 45, 3.4399999999999999, 0, 0.437, 6.556, 29.100000000000001, 4.5667, 5, 398, 15.199999999999999, 382.83999999999997, 4.5599999999999996, 29.800000000000001);
+INSERT INTO boston VALUES (193, 0.086639999999999995, 45, 3.4399999999999999, 0, 0.437, 7.1779999999999999, 26.300000000000001, 6.4798, 5, 398, 15.199999999999999, 390.49000000000001, 2.8700000000000001, 36.399999999999999);
+INSERT INTO boston VALUES (197, 0.04011, 80, 1.52, 0, 0.40400000000000003, 7.2869999999999999, 34.100000000000001, 7.3090000000000002, 2, 329, 12.6, 396.89999999999998, 4.0800000000000001, 33.299999999999997);
+INSERT INTO boston VALUES (201, 0.017780000000000001, 95, 1.47, 0, 0.40300000000000002, 7.1349999999999998, 13.9, 7.6534000000000004, 3, 402, 17, 384.30000000000001, 4.4500000000000002, 32.899999999999999);
+INSERT INTO boston VALUES (205, 0.02009, 95, 2.6800000000000002, 0, 0.41610000000000003, 8.0340000000000007, 31.899999999999999, 5.1180000000000003, 4, 224, 14.699999999999999, 390.55000000000001, 2.8799999999999999, 50);
+INSERT INTO boston VALUES (209, 0.13586999999999999, 0, 10.59, 1, 0.48899999999999999, 6.0640000000000001, 59.100000000000001, 4.2392000000000003, 4, 277, 18.600000000000001, 381.31999999999999, 14.66, 24.399999999999999);
+INSERT INTO boston VALUES (213, 0.21718999999999999, 0, 10.59, 1, 0.48899999999999999, 5.8070000000000004, 53.799999999999997, 3.6526000000000001, 4, 277, 18.600000000000001, 390.94, 16.030000000000001, 22.399999999999999);
+INSERT INTO boston VALUES (217, 0.045600000000000002, 0, 13.890000000000001, 1, 0.55000000000000004, 5.8879999999999999, 56, 3.1120999999999999, 5, 276, 16.399999999999999, 392.80000000000001, 13.51, 23.300000000000001);
+INSERT INTO boston VALUES (221, 0.35809000000000002, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.9509999999999996, 88.5, 2.8616999999999999, 8, 307, 17.399999999999999, 391.69999999999999, 9.7100000000000009, 26.699999999999999);
+INSERT INTO boston VALUES (225, 0.31533, 0, 6.2000000000000002, 0, 0.504, 8.266, 78.299999999999997, 2.8944000000000001, 8, 307, 17.399999999999999, 385.05000000000001, 4.1399999999999997, 44.799999999999997);
+INSERT INTO boston VALUES (229, 0.29819000000000001, 0, 6.2000000000000002, 0, 0.504, 7.6859999999999999, 17, 3.3751000000000002, 8, 307, 17.399999999999999, 377.50999999999999, 3.9199999999999999, 46.700000000000003);
+INSERT INTO boston VALUES (234, 0.33146999999999999, 0, 6.2000000000000002, 0, 0.50700000000000001, 8.2469999999999999, 70.400000000000006, 3.6518999999999999, 8, 307, 17.399999999999999, 378.94999999999999, 3.9500000000000002, 48.299999999999997);
+INSERT INTO boston VALUES (238, 0.51183000000000001, 0, 6.2000000000000002, 0, 0.50700000000000001, 7.3579999999999997, 71.599999999999994, 4.1479999999999997, 8, 307, 17.399999999999999, 390.06999999999999, 4.7300000000000004, 31.5);
+INSERT INTO boston VALUES (242, 0.10612000000000001, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.0949999999999998, 65.099999999999994, 6.3361000000000001, 6, 300, 16.600000000000001, 394.62, 12.4, 20.100000000000001);
+INSERT INTO boston VALUES (246, 0.19133, 22, 5.8600000000000003, 0, 0.43099999999999999, 5.6050000000000004, 70.200000000000003, 7.9549000000000003, 7, 330, 19.100000000000001, 389.13, 18.460000000000001, 18.5);
+INSERT INTO boston VALUES (250, 0.19073000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.718, 17.5, 7.8265000000000002, 7, 330, 19.100000000000001, 393.74000000000001, 6.5599999999999996, 26.199999999999999);
+INSERT INTO boston VALUES (254, 0.36893999999999999, 22, 5.8600000000000003, 0, 0.43099999999999999, 8.2590000000000003, 8.4000000000000004, 8.9067000000000007, 7, 330, 19.100000000000001, 396.89999999999998, 3.54, 42.799999999999997);
+INSERT INTO boston VALUES (257, 0.01538, 90, 3.75, 0, 0.39400000000000002, 7.4539999999999997, 34.200000000000003, 6.3361000000000001, 3, 244, 15.9, 386.33999999999997, 3.1099999999999999, 44);
+INSERT INTO boston VALUES (261, 0.54010999999999998, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.2030000000000003, 81.799999999999997, 2.1120999999999999, 5, 264, 13, 392.80000000000001, 9.5899999999999999, 33.799999999999997);
+INSERT INTO boston VALUES (265, 0.55006999999999995, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.2060000000000004, 91.599999999999994, 1.9300999999999999, 5, 264, 13, 387.88999999999999, 8.0999999999999996, 36.5);
+INSERT INTO boston VALUES (269, 0.54049999999999998, 20, 3.9700000000000002, 0, 0.57499999999999996, 7.4699999999999998, 52.600000000000001, 2.8719999999999999, 5, 264, 13, 390.30000000000001, 3.1600000000000001, 43.5);
+INSERT INTO boston VALUES (273, 0.11459999999999999, 20, 6.96, 0, 0.46400000000000002, 6.5380000000000003, 58.700000000000003, 3.9175, 3, 223, 18.600000000000001, 394.95999999999998, 7.7300000000000004, 24.399999999999999);
+INSERT INTO boston VALUES (277, 0.10469000000000001, 40, 6.4100000000000001, 1, 0.44700000000000001, 7.2670000000000003, 49, 4.7872000000000003, 4, 254, 17.600000000000001, 389.25, 6.0499999999999998, 33.200000000000003);
+INSERT INTO boston VALUES (281, 0.035779999999999999, 20, 3.3300000000000001, 0, 0.44290000000000002, 7.8200000000000003, 64.5, 4.6947000000000001, 5, 216, 14.9, 387.31, 3.7599999999999998, 45.399999999999999);
+INSERT INTO boston VALUES (285, 0.0090600000000000003, 90, 2.9700000000000002, 0, 0.40000000000000002, 7.0880000000000001, 20.800000000000001, 7.3072999999999997, 1, 285, 15.300000000000001, 394.72000000000003, 7.8499999999999996, 32.200000000000003);
+INSERT INTO boston VALUES (289, 0.045900000000000003, 52.5, 5.3200000000000003, 0, 0.40500000000000003, 6.3150000000000004, 45.600000000000001, 7.3171999999999997, 6, 293, 16.600000000000001, 396.89999999999998, 7.5999999999999996, 22.300000000000001);
+INSERT INTO boston VALUES (293, 0.036150000000000002, 80, 4.9500000000000002, 0, 0.41099999999999998, 6.6299999999999999, 23.399999999999999, 5.1166999999999998, 4, 245, 19.199999999999999, 396.89999999999998, 4.7000000000000002, 27.899999999999999);
+INSERT INTO boston VALUES (297, 0.053719999999999997, 0, 13.92, 0, 0.437, 6.5490000000000004, 51, 5.9603999999999999, 4, 289, 16, 392.85000000000002, 7.3899999999999997, 27.100000000000001);
+INSERT INTO boston VALUES (301, 0.044170000000000001, 70, 2.2400000000000002, 0, 0.40000000000000002, 6.8710000000000004, 47.399999999999999, 7.8277999999999999, 5, 358, 14.800000000000001, 390.86000000000001, 6.0700000000000003, 24.800000000000001);
+INSERT INTO boston VALUES (305, 0.055149999999999998, 33, 2.1800000000000002, 0, 0.47199999999999998, 7.2359999999999998, 41.100000000000001, 4.0220000000000002, 7, 222, 18.399999999999999, 393.68000000000001, 6.9299999999999997, 36.100000000000001);
+INSERT INTO boston VALUES (309, 0.49297999999999997, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.6349999999999998, 82.5, 3.3174999999999999, 4, 304, 18.399999999999999, 396.89999999999998, 4.54, 22.800000000000001);
+INSERT INTO boston VALUES (313, 0.26168999999999998, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.0229999999999997, 90.400000000000006, 2.8340000000000001, 4, 304, 18.399999999999999, 396.30000000000001, 11.720000000000001, 19.399999999999999);
+INSERT INTO boston VALUES (317, 0.31827, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.9139999999999997, 83.200000000000003, 3.9986000000000002, 4, 304, 18.399999999999999, 390.69999999999999, 18.329999999999998, 17.800000000000001);
+INSERT INTO boston VALUES (321, 0.1676, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.4260000000000002, 52.299999999999997, 4.5404, 5, 287, 19.600000000000001, 396.89999999999998, 7.2000000000000002, 23.800000000000001);
+INSERT INTO boston VALUES (325, 0.34109, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.415, 40.100000000000001, 4.7210999999999999, 5, 287, 19.600000000000001, 396.89999999999998, 6.1200000000000001, 25);
+INSERT INTO boston VALUES (329, 0.066170000000000007, 0, 3.2400000000000002, 0, 0.46000000000000002, 5.8680000000000003, 25.800000000000001, 5.2145999999999999, 4, 430, 16.899999999999999, 382.44, 9.9700000000000006, 19.300000000000001);
+INSERT INTO boston VALUES (333, 0.034660000000000003, 35, 6.0599999999999996, 0, 0.43790000000000001, 6.0309999999999997, 23.300000000000001, 6.6406999999999998, 1, 304, 16.899999999999999, 362.25, 7.8300000000000001, 19.399999999999999);
+INSERT INTO boston VALUES (337, 0.034270000000000002, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.8689999999999998, 46.299999999999997, 5.2310999999999996, 5, 224, 20.199999999999999, 396.89999999999998, 9.8000000000000007, 19.5);
+INSERT INTO boston VALUES (341, 0.061510000000000002, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.968, 58.5, 4.8121999999999998, 5, 224, 20.199999999999999, 396.89999999999998, 9.2899999999999991, 18.699999999999999);
+INSERT INTO boston VALUES (345, 0.03049, 55, 3.7799999999999998, 0, 0.48399999999999999, 6.8739999999999997, 28.100000000000001, 6.4653999999999998, 5, 370, 17.600000000000001, 387.97000000000003, 4.6100000000000003, 31.199999999999999);
+INSERT INTO boston VALUES (349, 0.015010000000000001, 80, 2.0099999999999998, 0, 0.435, 6.6349999999999998, 29.699999999999999, 8.3439999999999994, 4, 280, 17, 390.94, 5.9900000000000002, 24.5);
+INSERT INTO boston VALUES (353, 0.072440000000000004, 60, 1.6899999999999999, 0, 0.41099999999999998, 5.8840000000000003, 18.5, 10.7103, 4, 411, 18.300000000000001, 392.32999999999998, 7.79, 18.600000000000001);
+INSERT INTO boston VALUES (357, 8.9829600000000003, 0, 18.100000000000001, 1, 0.77000000000000002, 6.2119999999999997, 97.400000000000006, 2.1221999999999999, 24, 666, 20.199999999999999, 377.73000000000002, 17.600000000000001, 17.800000000000001);
+INSERT INTO boston VALUES (361, 4.5419200000000002, 0, 18.100000000000001, 0, 0.77000000000000002, 6.3979999999999997, 88, 2.5182000000000002, 24, 666, 20.199999999999999, 374.56, 7.79, 25);
+INSERT INTO boston VALUES (365, 3.4742799999999998, 0, 18.100000000000001, 1, 0.71799999999999997, 8.7799999999999994, 82.900000000000006, 1.9047000000000001, 24, 666, 20.199999999999999, 354.55000000000001, 5.29, 21.899999999999999);
+INSERT INTO boston VALUES (369, 4.8982200000000002, 0, 18.100000000000001, 0, 0.63100000000000001, 4.9699999999999998, 100, 1.3325, 24, 666, 20.199999999999999, 375.51999999999998, 3.2599999999999998, 50);
+INSERT INTO boston VALUES (373, 8.2672500000000007, 0, 18.100000000000001, 1, 0.66800000000000004, 5.875, 89.599999999999994, 1.1295999999999999, 24, 666, 20.199999999999999, 347.88, 8.8800000000000008, 50);
+INSERT INTO boston VALUES (377, 15.288, 0, 18.100000000000001, 0, 0.67100000000000004, 6.649, 93.299999999999997, 1.3449, 24, 666, 20.199999999999999, 363.01999999999998, 23.239999999999998, 13.9);
+INSERT INTO boston VALUES (381, 88.976200000000006, 0, 18.100000000000001, 0, 0.67100000000000004, 6.968, 91.900000000000006, 1.4165000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 17.210000000000001, 10.4);
+INSERT INTO boston VALUES (385, 20.084900000000001, 0, 18.100000000000001, 0, 0.69999999999999996, 4.3680000000000003, 91.200000000000003, 1.4395, 24, 666, 20.199999999999999, 285.82999999999998, 30.629999999999999, 8.8000000000000007);
+INSERT INTO boston VALUES (389, 14.3337, 0, 18.100000000000001, 0, 0.69999999999999996, 4.8799999999999999, 100, 1.5894999999999999, 24, 666, 20.199999999999999, 372.92000000000002, 30.620000000000001, 10.199999999999999);
+INSERT INTO boston VALUES (393, 11.5779, 0, 18.100000000000001, 0, 0.69999999999999996, 5.0359999999999996, 97, 1.77, 24, 666, 20.199999999999999, 396.89999999999998, 25.68, 9.6999999999999993);
+INSERT INTO boston VALUES (397, 5.8720499999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 6.4050000000000002, 96, 1.6768000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 19.370000000000001, 12.5);
+INSERT INTO boston VALUES (401, 25.046099999999999, 0, 18.100000000000001, 0, 0.69299999999999995, 5.9870000000000001, 100, 1.5888, 24, 666, 20.199999999999999, 396.89999999999998, 26.77, 5.5999999999999996);
+INSERT INTO boston VALUES (405, 41.529200000000003, 0, 18.100000000000001, 0, 0.69299999999999995, 5.5309999999999997, 85.400000000000006, 1.6073999999999999, 24, 666, 20.199999999999999, 329.45999999999998, 27.379999999999999, 8.5);
+INSERT INTO boston VALUES (409, 7.4038899999999996, 0, 18.100000000000001, 0, 0.59699999999999998, 5.617, 97.900000000000006, 1.4547000000000001, 24, 666, 20.199999999999999, 314.63999999999999, 26.399999999999999, 17.199999999999999);
+INSERT INTO boston VALUES (413, 18.811, 0, 18.100000000000001, 0, 0.59699999999999998, 4.6280000000000001, 100, 1.5539000000000001, 24, 666, 20.199999999999999, 28.789999999999999, 34.369999999999997, 17.899999999999999);
+INSERT INTO boston VALUES (417, 10.834199999999999, 0, 18.100000000000001, 0, 0.67900000000000005, 6.782, 90.799999999999997, 1.8194999999999999, 24, 666, 20.199999999999999, 21.57, 25.789999999999999, 7.5);
+INSERT INTO boston VALUES (421, 11.087400000000001, 0, 18.100000000000001, 0, 0.71799999999999997, 6.4109999999999996, 100, 1.8589, 24, 666, 20.199999999999999, 318.75, 15.02, 16.699999999999999);
+INSERT INTO boston VALUES (425, 8.7921200000000006, 0, 18.100000000000001, 0, 0.58399999999999996, 5.5650000000000004, 70.599999999999994, 2.0634999999999999, 24, 666, 20.199999999999999, 3.6499999999999999, 17.16, 11.699999999999999);
+INSERT INTO boston VALUES (429, 7.3671100000000003, 0, 18.100000000000001, 0, 0.67900000000000005, 6.1929999999999996, 78.099999999999994, 1.9356, 24, 666, 20.199999999999999, 96.730000000000004, 21.52, 11);
+INSERT INTO boston VALUES (433, 6.4440499999999998, 0, 18.100000000000001, 0, 0.58399999999999996, 6.4249999999999998, 74.799999999999997, 2.2004000000000001, 24, 666, 20.199999999999999, 97.950000000000003, 12.029999999999999, 16.100000000000001);
+INSERT INTO boston VALUES (437, 14.4208, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4610000000000003, 93.299999999999997, 2.0026000000000002, 24, 666, 20.199999999999999, 27.489999999999998, 18.050000000000001, 9.5999999999999996);
+INSERT INTO boston VALUES (441, 22.051100000000002, 0, 18.100000000000001, 0, 0.73999999999999999, 5.8179999999999996, 92.400000000000006, 1.8662000000000001, 24, 666, 20.199999999999999, 391.44999999999999, 22.109999999999999, 10.5);
+INSERT INTO boston VALUES (445, 12.802300000000001, 0, 18.100000000000001, 0, 0.73999999999999999, 5.8540000000000001, 96.599999999999994, 1.8956, 24, 666, 20.199999999999999, 240.52000000000001, 23.789999999999999, 10.800000000000001);
+INSERT INTO boston VALUES (449, 9.3290900000000008, 0, 18.100000000000001, 0, 0.71299999999999997, 6.1849999999999996, 98.700000000000003, 2.2616000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 18.129999999999999, 14.1);
+INSERT INTO boston VALUES (453, 5.0901699999999996, 0, 18.100000000000001, 0, 0.71299999999999997, 6.2969999999999997, 91.799999999999997, 2.3681999999999999, 24, 666, 20.199999999999999, 385.08999999999997, 17.27, 16.100000000000001);
+INSERT INTO boston VALUES (457, 4.6688299999999998, 0, 18.100000000000001, 0, 0.71299999999999997, 5.976, 87.900000000000006, 2.5806, 24, 666, 20.199999999999999, 10.48, 19.010000000000002, 12.699999999999999);
+INSERT INTO boston VALUES (461, 4.8121299999999998, 0, 18.100000000000001, 0, 0.71299999999999997, 6.7009999999999996, 90, 2.5975000000000001, 24, 666, 20.199999999999999, 255.22999999999999, 16.420000000000002, 16.399999999999999);
+INSERT INTO boston VALUES (3, 0.027289999999999998, 0, 7.0700000000000003, 0, 0.46899999999999997, 7.1849999999999996, 61.100000000000001, 4.9671000000000003, 2, 242, 17.800000000000001, 392.82999999999998, 4.0300000000000002, 34.700000000000003);
+INSERT INTO boston VALUES (7, 0.088289999999999993, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.0119999999999996, 66.599999999999994, 5.5605000000000002, 5, 311, 15.199999999999999, 395.60000000000002, 12.43, 22.899999999999999);
+INSERT INTO boston VALUES (11, 0.22489000000000001, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.3769999999999998, 94.299999999999997, 6.3467000000000002, 5, 311, 15.199999999999999, 392.51999999999998, 20.449999999999999, 15);
+INSERT INTO boston VALUES (15, 0.63795999999999997, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0960000000000001, 84.5, 4.4619, 4, 307, 21, 380.01999999999998, 10.26, 18.199999999999999);
+INSERT INTO boston VALUES (19, 0.80271000000000003, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.4560000000000004, 36.600000000000001, 3.7965, 4, 307, 21, 288.99000000000001, 11.69, 20.199999999999999);
+INSERT INTO boston VALUES (23, 1.23247, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.1420000000000003, 91.700000000000003, 3.9769000000000001, 4, 307, 21, 396.89999999999998, 18.719999999999999, 15.199999999999999);
+INSERT INTO boston VALUES (27, 0.67191000000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.8129999999999997, 90.299999999999997, 4.6820000000000004, 4, 307, 21, 376.88, 14.81, 16.600000000000001);
+INSERT INTO boston VALUES (31, 1.1308100000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.7130000000000001, 94.099999999999994, 4.2329999999999997, 4, 307, 21, 360.17000000000002, 22.600000000000001, 12.699999999999999);
+INSERT INTO boston VALUES (35, 1.6128199999999999, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0960000000000001, 96.900000000000006, 3.7597999999999998, 4, 307, 21, 248.31, 20.34, 13.5);
+INSERT INTO boston VALUES (39, 0.17505000000000001, 0, 5.96, 0, 0.499, 5.9660000000000002, 30.199999999999999, 3.8473000000000002, 5, 279, 19.199999999999999, 393.43000000000001, 10.130000000000001, 24.699999999999999);
+INSERT INTO boston VALUES (43, 0.14149999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.1689999999999996, 6.5999999999999996, 5.7209000000000003, 3, 233, 17.899999999999999, 383.37, 5.8099999999999996, 25.300000000000001);
+INSERT INTO boston VALUES (47, 0.18836, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.7859999999999996, 33.299999999999997, 5.1003999999999996, 3, 233, 17.899999999999999, 396.89999999999998, 14.15, 20);
+INSERT INTO boston VALUES (51, 0.088730000000000003, 21, 5.6399999999999997, 0, 0.439, 5.9630000000000001, 45.700000000000003, 6.8147000000000002, 4, 243, 16.800000000000001, 395.56, 13.449999999999999, 19.699999999999999);
+INSERT INTO boston VALUES (55, 0.013599999999999999, 75, 4, 0, 0.40999999999999998, 5.8879999999999999, 47.600000000000001, 7.3197000000000001, 3, 469, 21.100000000000001, 396.89999999999998, 14.800000000000001, 18.899999999999999);
+INSERT INTO boston VALUES (59, 0.15445, 25, 5.1299999999999999, 0, 0.45300000000000001, 6.1449999999999996, 29.199999999999999, 7.8148, 8, 284, 19.699999999999999, 390.68000000000001, 6.8600000000000003, 23.300000000000001);
+INSERT INTO boston VALUES (63, 0.11027000000000001, 25, 5.1299999999999999, 0, 0.45300000000000001, 6.4560000000000004, 67.799999999999997, 7.2255000000000003, 8, 284, 19.699999999999999, 396.89999999999998, 6.7300000000000004, 22.199999999999999);
+INSERT INTO boston VALUES (67, 0.043790000000000003, 80, 3.3700000000000001, 0, 0.39800000000000002, 5.7869999999999999, 31.100000000000001, 6.6115000000000004, 4, 337, 16.100000000000001, 396.89999999999998, 10.24, 19.399999999999999);
+INSERT INTO boston VALUES (71, 0.088260000000000005, 0, 10.81, 0, 0.41299999999999998, 6.4169999999999998, 6.5999999999999996, 5.2873000000000001, 4, 305, 19.199999999999999, 383.73000000000002, 6.7199999999999998, 24.199999999999999);
+INSERT INTO boston VALUES (75, 0.078960000000000002, 0, 12.83, 0, 0.437, 6.2729999999999997, 6, 4.2515000000000001, 5, 398, 18.699999999999999, 394.92000000000002, 6.7800000000000002, 24.100000000000001);
+INSERT INTO boston VALUES (79, 0.056460000000000003, 0, 12.83, 0, 0.437, 6.2320000000000002, 53.700000000000003, 5.0141, 5, 398, 18.699999999999999, 386.39999999999998, 12.34, 21.199999999999999);
+INSERT INTO boston VALUES (83, 0.036589999999999998, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.3019999999999996, 32.200000000000003, 5.4006999999999996, 4, 281, 19, 396.89999999999998, 6.7199999999999998, 24.800000000000001);
+INSERT INTO boston VALUES (87, 0.051880000000000003, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.0149999999999997, 45.100000000000001, 4.4272, 3, 247, 18.5, 395.99000000000001, 12.859999999999999, 22.5);
+INSERT INTO boston VALUES (91, 0.04684, 0, 3.4100000000000001, 0, 0.48899999999999999, 6.4169999999999998, 66.099999999999994, 3.0922999999999998, 2, 270, 17.800000000000001, 392.18000000000001, 8.8100000000000005, 22.600000000000001);
+INSERT INTO boston VALUES (95, 0.042939999999999999, 28, 15.039999999999999, 0, 0.46400000000000002, 6.2489999999999997, 77.299999999999997, 3.6150000000000002, 4, 270, 18.199999999999999, 396.89999999999998, 10.59, 20.600000000000001);
+INSERT INTO boston VALUES (99, 0.081869999999999998, 0, 2.8900000000000001, 0, 0.44500000000000001, 7.8200000000000003, 36.899999999999999, 3.4952000000000001, 2, 276, 18, 393.52999999999997, 3.5699999999999998, 43.799999999999997);
+INSERT INTO boston VALUES (103, 0.22875999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.4050000000000002, 85.400000000000006, 2.7147000000000001, 5, 384, 20.899999999999999, 70.799999999999997, 10.630000000000001, 18.600000000000001);
+INSERT INTO boston VALUES (107, 0.17119999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 5.8360000000000003, 91.900000000000006, 2.2109999999999999, 5, 384, 20.899999999999999, 395.67000000000002, 18.66, 19.5);
+INSERT INTO boston VALUES (111, 0.10793, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1950000000000003, 54.399999999999999, 2.7778, 5, 384, 20.899999999999999, 393.49000000000001, 13, 21.699999999999999);
+INSERT INTO boston VALUES (115, 0.14230999999999999, 0, 10.01, 0, 0.54700000000000004, 6.2539999999999996, 84.200000000000003, 2.2565, 6, 432, 17.800000000000001, 388.74000000000001, 10.449999999999999, 18.5);
+INSERT INTO boston VALUES (119, 0.13058, 0, 10.01, 0, 0.54700000000000004, 5.8719999999999999, 73.099999999999994, 2.4775, 6, 432, 17.800000000000001, 338.63, 15.369999999999999, 20.399999999999999);
+INSERT INTO boston VALUES (123, 0.092990000000000003, 0, 25.649999999999999, 0, 0.58099999999999996, 5.9610000000000003, 92.900000000000006, 2.0869, 2, 188, 19.100000000000001, 378.08999999999997, 17.93, 20.5);
+INSERT INTO boston VALUES (127, 0.38735000000000003, 0, 25.649999999999999, 0, 0.58099999999999996, 5.6130000000000004, 95.599999999999994, 1.7572000000000001, 2, 188, 19.100000000000001, 359.29000000000002, 27.260000000000002, 15.699999999999999);
+INSERT INTO boston VALUES (131, 0.34005999999999997, 0, 21.890000000000001, 0, 0.624, 6.4580000000000002, 98.900000000000006, 2.1185, 4, 437, 21.199999999999999, 395.04000000000002, 12.6, 19.199999999999999);
+INSERT INTO boston VALUES (135, 0.97616999999999998, 0, 21.890000000000001, 0, 0.624, 5.7569999999999997, 98.400000000000006, 2.3460000000000001, 4, 437, 21.199999999999999, 262.75999999999999, 17.309999999999999, 15.6);
+INSERT INTO boston VALUES (139, 0.24979999999999999, 0, 21.890000000000001, 0, 0.624, 5.8570000000000002, 98.200000000000003, 1.6686000000000001, 4, 437, 21.199999999999999, 392.04000000000002, 21.32, 13.300000000000001);
+INSERT INTO boston VALUES (143, 3.3210500000000001, 0, 19.579999999999998, 1, 0.871, 5.4029999999999996, 100, 1.3216000000000001, 5, 403, 14.699999999999999, 396.89999999999998, 26.82, 13.4);
+INSERT INTO boston VALUES (147, 2.1550500000000001, 0, 19.579999999999998, 0, 0.871, 5.6280000000000001, 100, 1.5165999999999999, 5, 403, 14.699999999999999, 169.27000000000001, 16.649999999999999, 15.6);
+INSERT INTO boston VALUES (151, 1.6566000000000001, 0, 19.579999999999998, 0, 0.871, 6.1219999999999999, 97.299999999999997, 1.6180000000000001, 5, 403, 14.699999999999999, 372.80000000000001, 14.1, 21.5);
+INSERT INTO boston VALUES (155, 1.4138500000000001, 0, 19.579999999999998, 1, 0.871, 6.1289999999999996, 96, 1.7494000000000001, 5, 403, 14.699999999999999, 321.01999999999998, 15.119999999999999, 17);
+INSERT INTO boston VALUES (159, 1.34284, 0, 19.579999999999998, 0, 0.60499999999999998, 6.0659999999999998, 100, 1.7573000000000001, 5, 403, 14.699999999999999, 353.88999999999999, 6.4299999999999997, 24.300000000000001);
+INSERT INTO boston VALUES (163, 1.8337699999999999, 0, 19.579999999999998, 1, 0.60499999999999998, 7.8019999999999996, 98.200000000000003, 2.0407000000000002, 5, 403, 14.699999999999999, 389.61000000000001, 1.9199999999999999, 50);
+INSERT INTO boston VALUES (167, 2.0101900000000001, 0, 19.579999999999998, 0, 0.60499999999999998, 7.9290000000000003, 96.200000000000003, 2.0459000000000001, 5, 403, 14.699999999999999, 369.30000000000001, 3.7000000000000002, 50);
+INSERT INTO boston VALUES (171, 1.2074199999999999, 0, 19.579999999999998, 0, 0.60499999999999998, 5.875, 94.599999999999994, 2.4258999999999999, 5, 403, 14.699999999999999, 292.29000000000002, 14.43, 17.399999999999999);
+INSERT INTO boston VALUES (175, 0.084470000000000003, 0, 4.0499999999999998, 0, 0.51000000000000001, 5.859, 68.700000000000003, 2.7019000000000002, 5, 296, 16.600000000000001, 393.23000000000002, 9.6400000000000006, 22.600000000000001);
+INSERT INTO boston VALUES (179, 0.066420000000000007, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.8600000000000003, 74.400000000000006, 2.9152999999999998, 5, 296, 16.600000000000001, 391.26999999999998, 6.9199999999999999, 29.899999999999999);
+INSERT INTO boston VALUES (183, 0.09103, 0, 2.46, 0, 0.48799999999999999, 7.1550000000000002, 92.200000000000003, 2.7006000000000001, 3, 193, 17.800000000000001, 394.12, 4.8200000000000003, 37.899999999999999);
+INSERT INTO boston VALUES (187, 0.05602, 0, 2.46, 0, 0.48799999999999999, 7.8310000000000004, 53.600000000000001, 3.1991999999999998, 3, 193, 17.800000000000001, 392.63, 4.4500000000000002, 50);
+INSERT INTO boston VALUES (191, 0.090679999999999997, 45, 3.4399999999999999, 0, 0.437, 6.9509999999999996, 21.5, 6.4798, 5, 398, 15.199999999999999, 377.68000000000001, 5.0999999999999996, 37);
+INSERT INTO boston VALUES (195, 0.01439, 60, 2.9300000000000002, 0, 0.40100000000000002, 6.6040000000000001, 18.800000000000001, 6.2195999999999998, 1, 265, 15.6, 376.69999999999999, 4.3799999999999999, 29.100000000000001);
+INSERT INTO boston VALUES (199, 0.037679999999999998, 80, 1.52, 0, 0.40400000000000003, 7.274, 38.299999999999997, 7.3090000000000002, 2, 329, 12.6, 392.19999999999999, 6.6200000000000001, 34.600000000000001);
+INSERT INTO boston VALUES (203, 0.021770000000000001, 82.5, 2.0299999999999998, 0, 0.41499999999999998, 7.6100000000000003, 15.699999999999999, 6.2699999999999996, 2, 348, 14.699999999999999, 395.38, 3.1099999999999999, 42.299999999999997);
+INSERT INTO boston VALUES (207, 0.22969000000000001, 0, 10.59, 0, 0.48899999999999999, 6.3259999999999996, 52.5, 4.3548999999999998, 4, 277, 18.600000000000001, 394.87, 10.970000000000001, 24.399999999999999);
+INSERT INTO boston VALUES (211, 0.17446, 0, 10.59, 1, 0.48899999999999999, 5.96, 92.099999999999994, 3.8771, 4, 277, 18.600000000000001, 393.25, 17.27, 21.699999999999999);
+INSERT INTO boston VALUES (215, 0.28954999999999997, 0, 10.59, 0, 0.48899999999999999, 5.4119999999999999, 9.8000000000000007, 3.5874999999999999, 4, 277, 18.600000000000001, 348.93000000000001, 29.550000000000001, 23.699999999999999);
+INSERT INTO boston VALUES (219, 0.11069, 0, 13.890000000000001, 1, 0.55000000000000004, 5.9509999999999996, 93.799999999999997, 2.8893, 5, 276, 16.399999999999999, 396.89999999999998, 17.920000000000002, 21.5);
+INSERT INTO boston VALUES (223, 0.62356, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.8789999999999996, 77.700000000000003, 3.2721, 8, 307, 17.399999999999999, 390.38999999999999, 9.9299999999999997, 27.5);
+INSERT INTO boston VALUES (227, 0.38213999999999998, 0, 6.2000000000000002, 0, 0.504, 8.0399999999999991, 86.5, 3.2157, 8, 307, 17.399999999999999, 387.38, 3.1299999999999999, 37.600000000000001);
+INSERT INTO boston VALUES (231, 0.53700000000000003, 0, 6.2000000000000002, 0, 0.504, 5.9809999999999999, 68.099999999999994, 3.6715, 8, 307, 17.399999999999999, 378.35000000000002, 11.65, 24.300000000000001);
+INSERT INTO boston VALUES (4, 0.032370000000000003, 0, 2.1800000000000002, 0, 0.45800000000000002, 6.9980000000000002, 45.799999999999997, 6.0621999999999998, 3, 222, 18.699999999999999, 394.63, 2.9399999999999999, 33.399999999999999);
+INSERT INTO boston VALUES (8, 0.14455000000000001, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.1719999999999997, 96.099999999999994, 5.9504999999999999, 5, 311, 15.199999999999999, 396.89999999999998, 19.149999999999999, 27.100000000000001);
+INSERT INTO boston VALUES (12, 0.11747, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.0090000000000003, 82.900000000000006, 6.2267000000000001, 5, 311, 15.199999999999999, 396.89999999999998, 13.27, 18.899999999999999);
+INSERT INTO boston VALUES (16, 0.62739, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.8339999999999996, 56.5, 4.4985999999999997, 4, 307, 21, 395.62, 8.4700000000000006, 19.899999999999999);
+INSERT INTO boston VALUES (20, 0.7258, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.7270000000000003, 69.5, 3.7965, 4, 307, 21, 390.94999999999999, 11.279999999999999, 18.199999999999999);
+INSERT INTO boston VALUES (24, 0.98843000000000003, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.8129999999999997, 100, 4.0952000000000002, 4, 307, 21, 394.54000000000002, 19.879999999999999, 14.5);
+INSERT INTO boston VALUES (28, 0.95577000000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0469999999999997, 88.799999999999997, 4.4534000000000002, 4, 307, 21, 306.38, 17.280000000000001, 14.800000000000001);
+INSERT INTO boston VALUES (32, 1.3547199999999999, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0720000000000001, 100, 4.1749999999999998, 4, 307, 21, 376.73000000000002, 13.039999999999999, 14.5);
+INSERT INTO boston VALUES (36, 0.064170000000000005, 0, 5.96, 0, 0.499, 5.9329999999999998, 68.200000000000003, 3.3603000000000001, 5, 279, 19.199999999999999, 396.89999999999998, 9.6799999999999997, 18.899999999999999);
+INSERT INTO boston VALUES (40, 0.027629999999999998, 75, 2.9500000000000002, 0, 0.42799999999999999, 6.5949999999999998, 21.800000000000001, 5.4010999999999996, 3, 252, 18.300000000000001, 395.63, 4.3200000000000003, 30.800000000000001);
+INSERT INTO boston VALUES (44, 0.15936, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.2110000000000003, 6.5, 5.7209000000000003, 3, 233, 17.899999999999999, 394.45999999999998, 7.4400000000000004, 24.699999999999999);
+INSERT INTO boston VALUES (48, 0.22927, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.0300000000000002, 85.5, 5.6894, 3, 233, 17.899999999999999, 392.74000000000001, 18.800000000000001, 16.600000000000001);
+INSERT INTO boston VALUES (52, 0.043369999999999999, 21, 5.6399999999999997, 0, 0.439, 6.1150000000000002, 63, 6.8147000000000002, 4, 243, 16.800000000000001, 393.97000000000003, 9.4299999999999997, 20.5);
+INSERT INTO boston VALUES (56, 0.01311, 90, 1.22, 0, 0.40300000000000002, 7.2489999999999997, 21.899999999999999, 8.6966000000000001, 5, 226, 17.899999999999999, 395.93000000000001, 4.8099999999999996, 35.399999999999999);
+INSERT INTO boston VALUES (60, 0.10328, 25, 5.1299999999999999, 0, 0.45300000000000001, 5.9269999999999996, 47.200000000000003, 6.9320000000000004, 8, 284, 19.699999999999999, 396.89999999999998, 9.2200000000000006, 19.600000000000001);
+INSERT INTO boston VALUES (64, 0.1265, 25, 5.1299999999999999, 0, 0.45300000000000001, 6.7619999999999996, 43.399999999999999, 7.9809000000000001, 8, 284, 19.699999999999999, 395.57999999999998, 9.5, 25);
+INSERT INTO boston VALUES (68, 0.057889999999999997, 12.5, 6.0700000000000003, 0, 0.40899999999999997, 5.8780000000000001, 21.399999999999999, 6.4980000000000002, 4, 345, 18.899999999999999, 396.20999999999998, 8.0999999999999996, 22);
+INSERT INTO boston VALUES (72, 0.15876000000000001, 0, 10.81, 0, 0.41299999999999998, 5.9610000000000003, 17.5, 5.2873000000000001, 4, 305, 19.199999999999999, 376.94, 9.8800000000000008, 21.699999999999999);
+INSERT INTO boston VALUES (76, 0.095119999999999996, 0, 12.83, 0, 0.437, 6.2859999999999996, 45, 4.5026000000000002, 5, 398, 18.699999999999999, 383.23000000000002, 8.9399999999999995, 21.399999999999999);
+INSERT INTO boston VALUES (80, 0.08387, 0, 12.83, 0, 0.437, 5.8739999999999997, 36.600000000000001, 4.5026000000000002, 5, 398, 18.699999999999999, 396.06, 9.0999999999999996, 20.300000000000001);
+INSERT INTO boston VALUES (84, 0.03551, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.1669999999999998, 46.700000000000003, 5.4006999999999996, 4, 281, 19, 390.63999999999999, 7.5099999999999998, 22.899999999999999);
+INSERT INTO boston VALUES (88, 0.071510000000000004, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.1210000000000004, 56.799999999999997, 3.7475999999999998, 3, 247, 18.5, 395.14999999999998, 8.4399999999999995, 22.199999999999999);
+INSERT INTO boston VALUES (92, 0.039320000000000001, 0, 3.4100000000000001, 0, 0.48899999999999999, 6.4050000000000002, 73.900000000000006, 3.0920999999999998, 2, 270, 17.800000000000001, 393.55000000000001, 8.1999999999999993, 22);
+INSERT INTO boston VALUES (96, 0.12204, 0, 2.8900000000000001, 0, 0.44500000000000001, 6.625, 57.799999999999997, 3.4952000000000001, 2, 276, 18, 357.98000000000002, 6.6500000000000004, 28.399999999999999);
+INSERT INTO boston VALUES (100, 0.068599999999999994, 0, 2.8900000000000001, 0, 0.44500000000000001, 7.4160000000000004, 62.5, 3.4952000000000001, 2, 276, 18, 396.89999999999998, 6.1900000000000004, 33.200000000000003);
+INSERT INTO boston VALUES (104, 0.21160999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1369999999999996, 87.400000000000006, 2.7147000000000001, 5, 384, 20.899999999999999, 394.47000000000003, 13.44, 19.300000000000001);
+INSERT INTO boston VALUES (108, 0.13117000000000001, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1269999999999998, 85.200000000000003, 2.1223999999999998, 5, 384, 20.899999999999999, 387.69, 14.09, 20.399999999999999);
+INSERT INTO boston VALUES (112, 0.10084, 0, 10.01, 0, 0.54700000000000004, 6.7149999999999999, 81.599999999999994, 2.6775000000000002, 6, 432, 17.800000000000001, 395.58999999999997, 10.16, 22.800000000000001);
+INSERT INTO boston VALUES (116, 0.17133999999999999, 0, 10.01, 0, 0.54700000000000004, 5.9279999999999999, 88.200000000000003, 2.4630999999999998, 6, 432, 17.800000000000001, 344.91000000000003, 15.76, 18.300000000000001);
+INSERT INTO boston VALUES (120, 0.14476, 0, 10.01, 0, 0.54700000000000004, 5.7309999999999999, 65.200000000000003, 2.7591999999999999, 6, 432, 17.800000000000001, 391.5, 13.609999999999999, 19.300000000000001);
+INSERT INTO boston VALUES (124, 0.15038000000000001, 0, 25.649999999999999, 0, 0.58099999999999996, 5.8559999999999999, 97, 1.9443999999999999, 2, 188, 19.100000000000001, 370.31, 25.41, 17.300000000000001);
+INSERT INTO boston VALUES (128, 0.25914999999999999, 0, 21.890000000000001, 0, 0.624, 5.6929999999999996, 96, 1.7883, 4, 437, 21.199999999999999, 392.11000000000001, 17.190000000000001, 16.199999999999999);
+INSERT INTO boston VALUES (132, 1.1929399999999999, 0, 21.890000000000001, 0, 0.624, 6.3259999999999996, 97.700000000000003, 2.2709999999999999, 4, 437, 21.199999999999999, 396.89999999999998, 12.26, 19.600000000000001);
+INSERT INTO boston VALUES (136, 0.55778000000000005, 0, 21.890000000000001, 0, 0.624, 6.335, 98.200000000000003, 2.1107, 4, 437, 21.199999999999999, 394.67000000000002, 16.960000000000001, 18.100000000000001);
+INSERT INTO boston VALUES (140, 0.54452, 0, 21.890000000000001, 0, 0.624, 6.1509999999999998, 97.900000000000006, 1.6687000000000001, 4, 437, 21.199999999999999, 396.89999999999998, 18.460000000000001, 17.800000000000001);
+INSERT INTO boston VALUES (144, 4.0974000000000004, 0, 19.579999999999998, 0, 0.871, 5.468, 100, 1.4117999999999999, 5, 403, 14.699999999999999, 396.89999999999998, 26.420000000000002, 15.6);
+INSERT INTO boston VALUES (148, 2.3686199999999999, 0, 19.579999999999998, 0, 0.871, 4.9260000000000002, 95.700000000000003, 1.4608000000000001, 5, 403, 14.699999999999999, 391.70999999999998, 29.530000000000001, 14.6);
+INSERT INTO boston VALUES (152, 1.4963200000000001, 0, 19.579999999999998, 0, 0.871, 5.4039999999999999, 100, 1.5915999999999999, 5, 403, 14.699999999999999, 341.60000000000002, 13.279999999999999, 19.600000000000001);
+INSERT INTO boston VALUES (156, 3.5350100000000002, 0, 19.579999999999998, 1, 0.871, 6.1520000000000001, 82.599999999999994, 1.7455000000000001, 5, 403, 14.699999999999999, 88.010000000000005, 15.02, 15.6);
+INSERT INTO boston VALUES (160, 1.42502, 0, 19.579999999999998, 0, 0.871, 6.5099999999999998, 100, 1.7659, 5, 403, 14.699999999999999, 364.31, 7.3899999999999997, 23.300000000000001);
+INSERT INTO boston VALUES (164, 1.51902, 0, 19.579999999999998, 1, 0.60499999999999998, 8.375, 93.900000000000006, 2.1619999999999999, 5, 403, 14.699999999999999, 388.44999999999999, 3.3199999999999998, 50);
+INSERT INTO boston VALUES (168, 1.8002800000000001, 0, 19.579999999999998, 0, 0.60499999999999998, 5.8769999999999998, 79.200000000000003, 2.4258999999999999, 5, 403, 14.699999999999999, 227.61000000000001, 12.140000000000001, 23.800000000000001);
+INSERT INTO boston VALUES (172, 2.3138999999999998, 0, 19.579999999999998, 0, 0.60499999999999998, 5.8799999999999999, 97.299999999999997, 2.3887, 5, 403, 14.699999999999999, 348.13, 12.029999999999999, 19.100000000000001);
+INSERT INTO boston VALUES (176, 0.066640000000000005, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.5460000000000003, 33.100000000000001, 3.1322999999999999, 5, 296, 16.600000000000001, 390.95999999999998, 5.3300000000000001, 29.399999999999999);
+INSERT INTO boston VALUES (180, 0.057799999999999997, 0, 2.46, 0, 0.48799999999999999, 6.9800000000000004, 58.399999999999999, 2.8290000000000002, 3, 193, 17.800000000000001, 396.89999999999998, 5.04, 37.200000000000003);
+INSERT INTO boston VALUES (184, 0.10008, 0, 2.46, 0, 0.48799999999999999, 6.5629999999999997, 95.599999999999994, 2.847, 3, 193, 17.800000000000001, 396.89999999999998, 5.6799999999999997, 32.5);
+INSERT INTO boston VALUES (188, 0.078750000000000001, 45, 3.4399999999999999, 0, 0.437, 6.782, 41.100000000000001, 3.7886000000000002, 5, 398, 15.199999999999999, 393.87, 6.6799999999999997, 32);
+INSERT INTO boston VALUES (192, 0.069110000000000005, 45, 3.4399999999999999, 0, 0.437, 6.7389999999999999, 30.800000000000001, 6.4798, 5, 398, 15.199999999999999, 389.70999999999998, 4.6900000000000004, 30.5);
+INSERT INTO boston VALUES (196, 0.013809999999999999, 80, 0.46000000000000002, 0, 0.42199999999999999, 7.875, 32, 5.6483999999999996, 4, 255, 14.4, 394.23000000000002, 2.9700000000000002, 50);
+INSERT INTO boston VALUES (200, 0.0315, 95, 1.47, 0, 0.40300000000000002, 6.9749999999999996, 15.300000000000001, 7.6534000000000004, 3, 402, 17, 396.89999999999998, 4.5599999999999996, 34.899999999999999);
+INSERT INTO boston VALUES (204, 0.035099999999999999, 95, 2.6800000000000002, 0, 0.41610000000000003, 7.8529999999999998, 33.200000000000003, 5.1180000000000003, 4, 224, 14.699999999999999, 392.77999999999997, 3.8100000000000001, 48.5);
+INSERT INTO boston VALUES (208, 0.25198999999999999, 0, 10.59, 0, 0.48899999999999999, 5.7830000000000004, 72.700000000000003, 4.3548999999999998, 4, 277, 18.600000000000001, 389.43000000000001, 18.059999999999999, 22.5);
+INSERT INTO boston VALUES (212, 0.37578, 0, 10.59, 1, 0.48899999999999999, 5.4039999999999999, 88.599999999999994, 3.665, 4, 277, 18.600000000000001, 395.24000000000001, 23.98, 19.300000000000001);
+INSERT INTO boston VALUES (216, 0.19802, 0, 10.59, 0, 0.48899999999999999, 6.1820000000000004, 42.399999999999999, 3.9453999999999998, 4, 277, 18.600000000000001, 393.63, 9.4700000000000006, 25);
+INSERT INTO boston VALUES (220, 0.11425, 0, 13.890000000000001, 1, 0.55000000000000004, 6.3730000000000002, 92.400000000000006, 3.3633000000000002, 5, 276, 16.399999999999999, 393.74000000000001, 10.5, 23);
+INSERT INTO boston VALUES (224, 0.61470000000000002, 0, 6.2000000000000002, 0, 0.50700000000000001, 6.6180000000000003, 80.799999999999997, 3.2721, 8, 307, 17.399999999999999, 396.89999999999998, 7.5999999999999996, 30.100000000000001);
+INSERT INTO boston VALUES (228, 0.41238000000000002, 0, 6.2000000000000002, 0, 0.504, 7.1630000000000003, 79.900000000000006, 3.2157, 8, 307, 17.399999999999999, 372.07999999999998, 6.3600000000000003, 31.600000000000001);
+INSERT INTO boston VALUES (232, 0.46295999999999998, 0, 6.2000000000000002, 0, 0.504, 7.4119999999999999, 76.900000000000006, 3.6715, 8, 307, 17.399999999999999, 376.13999999999999, 5.25, 31.699999999999999);
+INSERT INTO boston VALUES (233, 0.57528999999999997, 0, 6.2000000000000002, 0, 0.50700000000000001, 8.3369999999999997, 73.299999999999997, 3.8384, 8, 307, 17.399999999999999, 385.91000000000003, 2.4700000000000002, 41.700000000000003);
+INSERT INTO boston VALUES (237, 0.52058000000000004, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.6310000000000002, 76.5, 4.1479999999999997, 8, 307, 17.399999999999999, 388.44999999999999, 9.5399999999999991, 25.100000000000001);
+INSERT INTO boston VALUES (241, 0.11329, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.8970000000000002, 54.299999999999997, 6.3361000000000001, 6, 300, 16.600000000000001, 391.25, 11.380000000000001, 22);
+INSERT INTO boston VALUES (245, 0.20608000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 5.593, 76.5, 7.9549000000000003, 7, 330, 19.100000000000001, 372.49000000000001, 12.5, 17.600000000000001);
+INSERT INTO boston VALUES (249, 0.16439000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.4329999999999998, 49.100000000000001, 7.8265000000000002, 7, 330, 19.100000000000001, 374.70999999999998, 9.5199999999999996, 24.5);
+INSERT INTO boston VALUES (253, 0.082210000000000005, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.9569999999999999, 6.7999999999999998, 8.9067000000000007, 7, 330, 19.100000000000001, 386.08999999999997, 3.5299999999999998, 29.600000000000001);
+INSERT INTO boston VALUES (258, 0.61153999999999997, 20, 3.9700000000000002, 0, 0.64700000000000002, 8.7040000000000006, 86.900000000000006, 1.8009999999999999, 5, 264, 13, 389.69999999999999, 5.1200000000000001, 50);
+INSERT INTO boston VALUES (262, 0.53412000000000004, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.5199999999999996, 89.400000000000006, 2.1398000000000001, 5, 264, 13, 388.37, 7.2599999999999998, 43.100000000000001);
+INSERT INTO boston VALUES (266, 0.76161999999999996, 20, 3.9700000000000002, 0, 0.64700000000000002, 5.5599999999999996, 62.799999999999997, 1.9864999999999999, 5, 264, 13, 392.39999999999998, 10.449999999999999, 22.800000000000001);
+INSERT INTO boston VALUES (270, 0.090649999999999994, 20, 6.96, 1, 0.46400000000000002, 5.9199999999999999, 61.5, 3.9175, 3, 223, 18.600000000000001, 391.33999999999997, 13.65, 20.699999999999999);
+INSERT INTO boston VALUES (274, 0.22187999999999999, 20, 6.96, 1, 0.46400000000000002, 7.6909999999999998, 51.799999999999997, 4.3665000000000003, 3, 223, 18.600000000000001, 390.76999999999998, 6.5800000000000001, 35.200000000000003);
+INSERT INTO boston VALUES (278, 0.061269999999999998, 40, 6.4100000000000001, 1, 0.44700000000000001, 6.8259999999999996, 27.600000000000001, 4.8628, 4, 254, 17.600000000000001, 393.44999999999999, 4.1600000000000001, 33.100000000000001);
+INSERT INTO boston VALUES (282, 0.03705, 20, 3.3300000000000001, 0, 0.44290000000000002, 6.968, 37.200000000000003, 5.2446999999999999, 5, 216, 14.9, 392.23000000000002, 4.5899999999999999, 35.399999999999999);
+INSERT INTO boston VALUES (286, 0.010959999999999999, 55, 2.25, 0, 0.38900000000000001, 6.4530000000000003, 31.899999999999999, 7.3072999999999997, 1, 300, 15.300000000000001, 394.72000000000003, 8.2300000000000004, 22);
+INSERT INTO boston VALUES (290, 0.042970000000000001, 52.5, 5.3200000000000003, 0, 0.40500000000000003, 6.5650000000000004, 22.899999999999999, 7.3171999999999997, 6, 293, 16.600000000000001, 371.72000000000003, 9.5099999999999998, 24.800000000000001);
+INSERT INTO boston VALUES (294, 0.082650000000000001, 0, 13.92, 0, 0.437, 6.1269999999999998, 18.399999999999999, 5.5026999999999999, 4, 289, 16, 396.89999999999998, 8.5800000000000001, 23.899999999999999);
+INSERT INTO boston VALUES (298, 0.14102999999999999, 0, 13.92, 0, 0.437, 5.79, 58, 6.3200000000000003, 4, 289, 16, 396.89999999999998, 15.84, 20.300000000000001);
+INSERT INTO boston VALUES (302, 0.035369999999999999, 34, 6.0899999999999999, 0, 0.433, 6.5899999999999999, 40.399999999999999, 5.4916999999999998, 7, 329, 16.100000000000001, 395.75, 9.5, 22);
+INSERT INTO boston VALUES (306, 0.054789999999999998, 33, 2.1800000000000002, 0, 0.47199999999999998, 6.6159999999999997, 58.100000000000001, 3.3700000000000001, 7, 222, 18.399999999999999, 393.36000000000001, 8.9299999999999997, 28.399999999999999);
+INSERT INTO boston VALUES (310, 0.34939999999999999, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.9720000000000004, 76.700000000000003, 3.1025, 4, 304, 18.399999999999999, 396.24000000000001, 9.9700000000000006, 20.300000000000001);
+INSERT INTO boston VALUES (314, 0.26938000000000001, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.266, 82.799999999999997, 3.2627999999999999, 4, 304, 18.399999999999999, 393.38999999999999, 7.9000000000000004, 21.600000000000001);
+INSERT INTO boston VALUES (318, 0.24521999999999999, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.782, 71.700000000000003, 4.0316999999999998, 4, 304, 18.399999999999999, 396.89999999999998, 15.94, 19.800000000000001);
+INSERT INTO boston VALUES (322, 0.18159, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.3760000000000003, 54.299999999999997, 4.5404, 5, 287, 19.600000000000001, 396.89999999999998, 6.8700000000000001, 23.100000000000001);
+INSERT INTO boston VALUES (326, 0.19186, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.431, 14.699999999999999, 5.4158999999999997, 5, 287, 19.600000000000001, 393.68000000000001, 5.0800000000000001, 24.600000000000001);
+INSERT INTO boston VALUES (330, 0.067239999999999994, 0, 3.2400000000000002, 0, 0.46000000000000002, 6.3330000000000002, 17.199999999999999, 5.2145999999999999, 4, 430, 16.899999999999999, 375.20999999999998, 7.3399999999999999, 22.600000000000001);
+INSERT INTO boston VALUES (334, 0.05083, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.3159999999999998, 38.100000000000001, 6.4584000000000001, 5, 224, 20.199999999999999, 389.70999999999998, 5.6799999999999997, 22.199999999999999);
+INSERT INTO boston VALUES (338, 0.03041, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.8949999999999996, 59.600000000000001, 5.6150000000000002, 5, 224, 20.199999999999999, 394.81, 10.56, 18.5);
+INSERT INTO boston VALUES (342, 0.013010000000000001, 35, 1.52, 0, 0.442, 7.2409999999999997, 49.299999999999997, 7.0378999999999996, 1, 284, 15.5, 394.74000000000001, 5.4900000000000002, 32.700000000000003);
+INSERT INTO boston VALUES (346, 0.031130000000000001, 0, 4.3899999999999997, 0, 0.442, 6.0140000000000002, 48.5, 8.0136000000000003, 3, 352, 18.800000000000001, 385.63999999999999, 10.529999999999999, 17.5);
+INSERT INTO boston VALUES (350, 0.028989999999999998, 40, 1.25, 0, 0.42899999999999999, 6.9390000000000001, 34.5, 8.7920999999999996, 1, 335, 19.699999999999999, 389.85000000000002, 5.8899999999999997, 26.600000000000001);
+INSERT INTO boston VALUES (354, 0.017090000000000001, 90, 2.02, 0, 0.40999999999999998, 6.7279999999999998, 36.100000000000001, 12.1265, 5, 187, 17, 384.45999999999998, 4.5, 30.100000000000001);
+INSERT INTO boston VALUES (358, 3.8496999999999999, 0, 18.100000000000001, 1, 0.77000000000000002, 6.3949999999999996, 91, 2.5051999999999999, 24, 666, 20.199999999999999, 391.33999999999997, 13.27, 21.699999999999999);
+INSERT INTO boston VALUES (362, 3.83684, 0, 18.100000000000001, 0, 0.77000000000000002, 6.2510000000000003, 91.099999999999994, 2.2955000000000001, 24, 666, 20.199999999999999, 350.64999999999998, 14.19, 19.899999999999999);
+INSERT INTO boston VALUES (366, 4.5558699999999996, 0, 18.100000000000001, 0, 0.71799999999999997, 3.5609999999999999, 87.900000000000006, 1.6132, 24, 666, 20.199999999999999, 354.69999999999999, 7.1200000000000001, 27.5);
+INSERT INTO boston VALUES (370, 5.6699799999999998, 0, 18.100000000000001, 1, 0.63100000000000001, 6.6829999999999998, 96.799999999999997, 1.3567, 24, 666, 20.199999999999999, 375.32999999999998, 3.73, 50);
+INSERT INTO boston VALUES (374, 11.1081, 0, 18.100000000000001, 0, 0.66800000000000004, 4.9059999999999997, 100, 1.1741999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 34.770000000000003, 13.800000000000001);
+INSERT INTO boston VALUES (378, 9.8234899999999996, 0, 18.100000000000001, 0, 0.67100000000000004, 6.7939999999999996, 98.799999999999997, 1.3580000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 21.239999999999998, 13.300000000000001);
+INSERT INTO boston VALUES (382, 15.8744, 0, 18.100000000000001, 0, 0.67100000000000004, 6.5449999999999999, 99.099999999999994, 1.5192000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 21.079999999999998, 10.9);
+INSERT INTO boston VALUES (386, 16.811800000000002, 0, 18.100000000000001, 0, 0.69999999999999996, 5.2770000000000001, 98.099999999999994, 1.4260999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 30.809999999999999, 7.2000000000000002);
+INSERT INTO boston VALUES (390, 8.1517400000000002, 0, 18.100000000000001, 0, 0.69999999999999996, 5.3899999999999997, 98.900000000000006, 1.7281, 24, 666, 20.199999999999999, 396.89999999999998, 20.850000000000001, 11.5);
+INSERT INTO boston VALUES (394, 8.6447599999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 6.1929999999999996, 92.599999999999994, 1.7911999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 15.17, 13.800000000000001);
+INSERT INTO boston VALUES (398, 7.6720199999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 5.7469999999999999, 98.900000000000006, 1.6334, 24, 666, 20.199999999999999, 393.10000000000002, 19.920000000000002, 8.5);
+INSERT INTO boston VALUES (402, 14.2362, 0, 18.100000000000001, 0, 0.69299999999999995, 6.343, 100, 1.5741000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 20.32, 7.2000000000000002);
+INSERT INTO boston VALUES (406, 67.9208, 0, 18.100000000000001, 0, 0.69299999999999995, 5.6829999999999998, 100, 1.4254, 24, 666, 20.199999999999999, 384.97000000000003, 22.98, 5);
+INSERT INTO boston VALUES (410, 14.4383, 0, 18.100000000000001, 0, 0.59699999999999998, 6.8520000000000003, 100, 1.4655, 24, 666, 20.199999999999999, 179.36000000000001, 19.780000000000001, 27.5);
+INSERT INTO boston VALUES (414, 28.655799999999999, 0, 18.100000000000001, 0, 0.59699999999999998, 5.1550000000000002, 100, 1.5893999999999999, 24, 666, 20.199999999999999, 210.97, 20.079999999999998, 16.300000000000001);
+INSERT INTO boston VALUES (418, 25.9406, 0, 18.100000000000001, 0, 0.67900000000000005, 5.3040000000000003, 89.099999999999994, 1.6475, 24, 666, 20.199999999999999, 127.36, 26.640000000000001, 10.4);
+INSERT INTO boston VALUES (422, 7.0225900000000001, 0, 18.100000000000001, 0, 0.71799999999999997, 6.0060000000000002, 95.299999999999997, 1.8746, 24, 666, 20.199999999999999, 319.98000000000002, 15.699999999999999, 14.199999999999999);
+INSERT INTO boston VALUES (426, 15.860300000000001, 0, 18.100000000000001, 0, 0.67900000000000005, 5.8959999999999999, 95.400000000000006, 1.9096, 24, 666, 20.199999999999999, 7.6799999999999997, 24.390000000000001, 8.3000000000000007);
+INSERT INTO boston VALUES (430, 9.3388899999999992, 0, 18.100000000000001, 0, 0.67900000000000005, 6.3799999999999999, 95.599999999999994, 1.9681999999999999, 24, 666, 20.199999999999999, 60.719999999999999, 24.079999999999998, 9.5);
+INSERT INTO boston VALUES (434, 5.5810700000000004, 0, 18.100000000000001, 0, 0.71299999999999997, 6.4359999999999999, 87.900000000000006, 2.3157999999999999, 24, 666, 20.199999999999999, 100.19, 16.219999999999999, 14.300000000000001);
+INSERT INTO boston VALUES (438, 15.177199999999999, 0, 18.100000000000001, 0, 0.73999999999999999, 6.1520000000000001, 100, 1.9141999999999999, 24, 666, 20.199999999999999, 9.3200000000000003, 26.449999999999999, 8.6999999999999993);
+INSERT INTO boston VALUES (442, 9.7241800000000005, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4059999999999997, 97.200000000000003, 2.0651000000000002, 24, 666, 20.199999999999999, 385.95999999999998, 19.52, 17.100000000000001);
+INSERT INTO boston VALUES (446, 10.671799999999999, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4589999999999996, 94.799999999999997, 1.9879, 24, 666, 20.199999999999999, 43.060000000000002, 23.98, 11.800000000000001);
+INSERT INTO boston VALUES (450, 7.5260100000000003, 0, 18.100000000000001, 0, 0.71299999999999997, 6.4169999999999998, 98.299999999999997, 2.1850000000000001, 24, 666, 20.199999999999999, 304.20999999999998, 19.309999999999999, 13);
+INSERT INTO boston VALUES (454, 8.2480899999999995, 0, 18.100000000000001, 0, 0.71299999999999997, 7.3929999999999998, 99.299999999999997, 2.4527000000000001, 24, 666, 20.199999999999999, 375.87, 16.739999999999998, 17.800000000000001);
+INSERT INTO boston VALUES (458, 8.2005800000000004, 0, 18.100000000000001, 0, 0.71299999999999997, 5.9359999999999999, 80.299999999999997, 2.7791999999999999, 24, 666, 20.199999999999999, 3.5, 16.940000000000001, 13.5);
+INSERT INTO boston VALUES (462, 3.6931099999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.3760000000000003, 88.400000000000006, 2.5670999999999999, 24, 666, 20.199999999999999, 391.43000000000001, 14.65, 17.699999999999999);
+INSERT INTO boston VALUES (465, 7.8393199999999998, 0, 18.100000000000001, 0, 0.65500000000000003, 6.2089999999999996, 65.400000000000006, 2.9634, 24, 666, 20.199999999999999, 396.89999999999998, 13.220000000000001, 21.399999999999999);
+INSERT INTO boston VALUES (469, 15.575699999999999, 0, 18.100000000000001, 0, 0.57999999999999996, 5.9260000000000002, 71, 2.9083999999999999, 24, 666, 20.199999999999999, 368.74000000000001, 18.129999999999999, 19.100000000000001);
+INSERT INTO boston VALUES (473, 3.5686800000000001, 0, 18.100000000000001, 0, 0.57999999999999996, 6.4370000000000003, 75, 2.8965000000000001, 24, 666, 20.199999999999999, 393.37, 14.359999999999999, 23.199999999999999);
+INSERT INTO boston VALUES (477, 4.87141, 0, 18.100000000000001, 0, 0.61399999999999999, 6.484, 93.599999999999994, 2.3052999999999999, 24, 666, 20.199999999999999, 396.20999999999998, 18.68, 16.699999999999999);
+INSERT INTO boston VALUES (481, 5.8240100000000004, 0, 18.100000000000001, 0, 0.53200000000000003, 6.242, 64.700000000000003, 3.4241999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 10.74, 23);
+INSERT INTO boston VALUES (485, 2.3785699999999999, 0, 18.100000000000001, 0, 0.58299999999999996, 5.8710000000000004, 41.899999999999999, 3.7240000000000002, 24, 666, 20.199999999999999, 370.73000000000002, 13.34, 20.600000000000001);
+INSERT INTO boston VALUES (489, 0.15085999999999999, 0, 27.739999999999998, 0, 0.60899999999999999, 5.4539999999999997, 92.700000000000003, 1.8209, 4, 711, 20.100000000000001, 395.08999999999997, 18.059999999999999, 15.199999999999999);
+INSERT INTO boston VALUES (493, 0.11132, 0, 27.739999999999998, 0, 0.60899999999999999, 5.9829999999999997, 83.5, 2.1099000000000001, 4, 711, 20.100000000000001, 396.89999999999998, 13.35, 20.100000000000001);
+INSERT INTO boston VALUES (497, 0.28960000000000002, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.3899999999999997, 72.900000000000006, 2.7986, 6, 391, 19.199999999999999, 396.89999999999998, 21.140000000000001, 19.699999999999999);
+INSERT INTO boston VALUES (501, 0.22438, 0, 9.6899999999999995, 0, 0.58499999999999996, 6.0270000000000001, 79.700000000000003, 2.4982000000000002, 6, 391, 19.199999999999999, 396.89999999999998, 14.33, 16.800000000000001);
+INSERT INTO boston VALUES (505, 0.10959000000000001, 0, 11.93, 0, 0.57299999999999995, 6.7939999999999996, 89.299999999999997, 2.3889, 1, 273, 21, 393.44999999999999, 6.4800000000000004, 22);
+INSERT INTO boston VALUES (235, 0.44790999999999997, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.726, 66.5, 3.6518999999999999, 8, 307, 17.399999999999999, 360.19999999999999, 8.0500000000000007, 29);
+INSERT INTO boston VALUES (239, 0.082439999999999999, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.4809999999999999, 18.5, 6.1898999999999997, 6, 300, 16.600000000000001, 379.41000000000003, 6.3600000000000003, 23.699999999999999);
+INSERT INTO boston VALUES (243, 0.10290000000000001, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.3579999999999997, 52.899999999999999, 7.0354999999999999, 6, 300, 16.600000000000001, 372.75, 11.220000000000001, 22.199999999999999);
+INSERT INTO boston VALUES (247, 0.33983000000000002, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.1079999999999997, 34.899999999999999, 8.0555000000000003, 7, 330, 19.100000000000001, 390.18000000000001, 9.1600000000000001, 24.300000000000001);
+INSERT INTO boston VALUES (251, 0.14030000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.4870000000000001, 13, 7.3967000000000001, 7, 330, 19.100000000000001, 396.27999999999997, 5.9000000000000004, 24.399999999999999);
+INSERT INTO boston VALUES (255, 0.048189999999999997, 80, 3.6400000000000001, 0, 0.39200000000000002, 6.1079999999999997, 32, 9.2202999999999999, 1, 315, 16.399999999999999, 392.88999999999999, 6.5700000000000003, 21.899999999999999);
+INSERT INTO boston VALUES (256, 0.035479999999999998, 80, 3.6400000000000001, 0, 0.39200000000000002, 5.8760000000000003, 19.100000000000001, 9.2202999999999999, 1, 315, 16.399999999999999, 395.18000000000001, 9.25, 20.899999999999999);
+INSERT INTO boston VALUES (260, 0.65664999999999996, 20, 3.9700000000000002, 0, 0.64700000000000002, 6.8419999999999996, 100, 2.0106999999999999, 5, 264, 13, 391.93000000000001, 6.9000000000000004, 30.100000000000001);
+INSERT INTO boston VALUES (264, 0.82525999999999999, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.327, 94.5, 2.0788000000000002, 5, 264, 13, 393.42000000000002, 11.25, 31);
+INSERT INTO boston VALUES (268, 0.57833999999999997, 20, 3.9700000000000002, 0, 0.57499999999999996, 8.2970000000000006, 67, 2.4216000000000002, 5, 264, 13, 384.54000000000002, 7.4400000000000004, 50);
+INSERT INTO boston VALUES (272, 0.16211, 20, 6.96, 0, 0.46400000000000002, 6.2400000000000002, 16.300000000000001, 4.4290000000000003, 3, 223, 18.600000000000001, 396.89999999999998, 6.5899999999999999, 25.199999999999999);
+INSERT INTO boston VALUES (276, 0.09604, 40, 6.4100000000000001, 0, 0.44700000000000001, 6.8540000000000001, 42.799999999999997, 4.2672999999999996, 4, 254, 17.600000000000001, 396.89999999999998, 2.98, 32);
+INSERT INTO boston VALUES (280, 0.21038000000000001, 20, 3.3300000000000001, 0, 0.44290000000000002, 6.8120000000000003, 32.200000000000003, 4.1006999999999998, 5, 216, 14.9, 396.89999999999998, 4.8499999999999996, 35.100000000000001);
+INSERT INTO boston VALUES (284, 0.015010000000000001, 90, 1.21, 1, 0.40100000000000002, 7.923, 24.800000000000001, 5.8849999999999998, 1, 198, 13.6, 395.51999999999998, 3.1600000000000001, 50);
+INSERT INTO boston VALUES (288, 0.038710000000000001, 52.5, 5.3200000000000003, 0, 0.40500000000000003, 6.2089999999999996, 31.300000000000001, 7.3171999999999997, 6, 293, 16.600000000000001, 396.89999999999998, 7.1399999999999997, 23.199999999999999);
+INSERT INTO boston VALUES (292, 0.07886, 80, 4.9500000000000002, 0, 0.41099999999999998, 7.1479999999999997, 27.699999999999999, 5.1166999999999998, 4, 245, 19.199999999999999, 396.89999999999998, 3.5600000000000001, 37.299999999999997);
+INSERT INTO boston VALUES (296, 0.12931999999999999, 0, 13.92, 0, 0.437, 6.6779999999999999, 31.100000000000001, 5.9603999999999999, 4, 289, 16, 396.89999999999998, 6.2699999999999996, 28.600000000000001);
+INSERT INTO boston VALUES (300, 0.05561, 70, 2.2400000000000002, 0, 0.40000000000000002, 7.0410000000000004, 10, 7.8277999999999999, 5, 358, 14.800000000000001, 371.57999999999998, 4.7400000000000002, 29);
+INSERT INTO boston VALUES (304, 0.10000000000000001, 34, 6.0899999999999999, 0, 0.433, 6.9820000000000002, 17.699999999999999, 5.4916999999999998, 7, 329, 16.100000000000001, 390.43000000000001, 4.8600000000000003, 33.100000000000001);
+INSERT INTO boston VALUES (308, 0.049320000000000003, 33, 2.1800000000000002, 0, 0.47199999999999998, 6.8490000000000002, 70.299999999999997, 3.1827000000000001, 7, 222, 18.399999999999999, 396.89999999999998, 7.5300000000000002, 28.199999999999999);
+INSERT INTO boston VALUES (312, 0.79040999999999995, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.1219999999999999, 52.799999999999997, 2.6402999999999999, 4, 304, 18.399999999999999, 396.89999999999998, 5.9800000000000004, 22.100000000000001);
+INSERT INTO boston VALUES (316, 0.25356000000000001, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.7050000000000001, 77.700000000000003, 3.9449999999999998, 4, 304, 18.399999999999999, 396.42000000000002, 11.5, 16.199999999999999);
+INSERT INTO boston VALUES (320, 0.47547, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.1130000000000004, 58.799999999999997, 4.0019, 4, 304, 18.399999999999999, 396.23000000000002, 12.73, 21);
+INSERT INTO boston VALUES (324, 0.28392000000000001, 0, 7.3799999999999999, 0, 0.49299999999999999, 5.7080000000000002, 74.299999999999997, 4.7210999999999999, 5, 287, 19.600000000000001, 391.13, 11.74, 18.5);
+INSERT INTO boston VALUES (328, 0.24102999999999999, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.0830000000000002, 43.700000000000003, 5.4158999999999997, 5, 287, 19.600000000000001, 396.89999999999998, 12.789999999999999, 22.199999999999999);
+INSERT INTO boston VALUES (332, 0.050229999999999997, 35, 6.0599999999999996, 0, 0.43790000000000001, 5.7060000000000004, 28.399999999999999, 6.6406999999999998, 1, 304, 16.899999999999999, 394.01999999999998, 12.43, 17.100000000000001);
+INSERT INTO boston VALUES (336, 0.039609999999999999, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.0369999999999999, 34.5, 5.9852999999999996, 5, 224, 20.199999999999999, 396.89999999999998, 8.0099999999999998, 21.100000000000001);
+INSERT INTO boston VALUES (340, 0.054969999999999998, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.9850000000000003, 45.399999999999999, 4.8121999999999998, 5, 224, 20.199999999999999, 396.89999999999998, 9.7400000000000002, 19);
+INSERT INTO boston VALUES (344, 0.025430000000000001, 55, 3.7799999999999998, 0, 0.48399999999999999, 6.6959999999999997, 56.399999999999999, 5.7321, 5, 370, 17.600000000000001, 396.89999999999998, 7.1799999999999997, 23.899999999999999);
+INSERT INTO boston VALUES (348, 0.018700000000000001, 85, 4.1500000000000004, 0, 0.42899999999999999, 6.516, 27.699999999999999, 8.5352999999999994, 4, 351, 17.899999999999999, 392.43000000000001, 6.3600000000000003, 23.100000000000001);
+INSERT INTO boston VALUES (352, 0.079500000000000001, 60, 1.6899999999999999, 0, 0.41099999999999998, 6.5789999999999997, 35.899999999999999, 10.7103, 4, 411, 18.300000000000001, 370.77999999999997, 5.4900000000000002, 24.100000000000001);
+INSERT INTO boston VALUES (356, 0.10659, 80, 1.9099999999999999, 0, 0.41299999999999998, 5.9359999999999999, 19.5, 10.585699999999999, 4, 334, 22, 376.04000000000002, 5.5700000000000003, 20.600000000000001);
+INSERT INTO boston VALUES (360, 4.2613099999999999, 0, 18.100000000000001, 0, 0.77000000000000002, 6.1120000000000001, 81.299999999999997, 2.5091000000000001, 24, 666, 20.199999999999999, 390.74000000000001, 12.67, 22.600000000000001);
+INSERT INTO boston VALUES (364, 4.2223899999999999, 0, 18.100000000000001, 1, 0.77000000000000002, 5.8029999999999999, 89, 1.9047000000000001, 24, 666, 20.199999999999999, 353.04000000000002, 14.640000000000001, 16.800000000000001);
+INSERT INTO boston VALUES (368, 13.5222, 0, 18.100000000000001, 0, 0.63100000000000001, 3.863, 100, 1.5105999999999999, 24, 666, 20.199999999999999, 131.41999999999999, 13.33, 23.100000000000001);
+INSERT INTO boston VALUES (372, 9.2323000000000004, 0, 18.100000000000001, 0, 0.63100000000000001, 6.2160000000000002, 100, 1.1691, 24, 666, 20.199999999999999, 366.14999999999998, 9.5299999999999994, 50);
+INSERT INTO boston VALUES (376, 19.609100000000002, 0, 18.100000000000001, 0, 0.67100000000000004, 7.3129999999999997, 97.900000000000006, 1.3163, 24, 666, 20.199999999999999, 396.89999999999998, 13.44, 15);
+INSERT INTO boston VALUES (380, 17.866700000000002, 0, 18.100000000000001, 0, 0.67100000000000004, 6.2229999999999999, 100, 1.3861000000000001, 24, 666, 20.199999999999999, 393.74000000000001, 21.780000000000001, 10.199999999999999);
+INSERT INTO boston VALUES (384, 7.9924799999999996, 0, 18.100000000000001, 0, 0.69999999999999996, 5.5199999999999996, 100, 1.5330999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 24.559999999999999, 12.300000000000001);
+INSERT INTO boston VALUES (388, 22.597100000000001, 0, 18.100000000000001, 0, 0.69999999999999996, 5, 89.5, 1.5184, 24, 666, 20.199999999999999, 396.89999999999998, 31.989999999999998, 7.4000000000000004);
+INSERT INTO boston VALUES (392, 5.29305, 0, 18.100000000000001, 0, 0.69999999999999996, 6.0510000000000002, 82.5, 2.1678000000000002, 24, 666, 20.199999999999999, 378.38, 18.760000000000002, 23.199999999999999);
+INSERT INTO boston VALUES (396, 8.7167499999999993, 0, 18.100000000000001, 0, 0.69299999999999995, 6.4710000000000001, 98.799999999999997, 1.7257, 24, 666, 20.199999999999999, 391.98000000000002, 17.120000000000001, 13.1);
+INSERT INTO boston VALUES (400, 9.9165500000000009, 0, 18.100000000000001, 0, 0.69299999999999995, 5.8520000000000003, 77.799999999999997, 1.5004, 24, 666, 20.199999999999999, 338.16000000000003, 29.969999999999999, 6.2999999999999998);
+INSERT INTO boston VALUES (404, 24.8017, 0, 18.100000000000001, 0, 0.69299999999999995, 5.3490000000000002, 96, 1.7028000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 19.77, 8.3000000000000007);
+INSERT INTO boston VALUES (408, 11.9511, 0, 18.100000000000001, 0, 0.65900000000000003, 5.6079999999999997, 100, 1.2851999999999999, 24, 666, 20.199999999999999, 332.08999999999997, 12.130000000000001, 27.899999999999999);
+INSERT INTO boston VALUES (412, 14.050700000000001, 0, 18.100000000000001, 0, 0.59699999999999998, 6.657, 100, 1.5275000000000001, 24, 666, 20.199999999999999, 35.049999999999997, 21.219999999999999, 17.199999999999999);
+INSERT INTO boston VALUES (416, 18.084599999999998, 0, 18.100000000000001, 0, 0.67900000000000005, 6.4340000000000002, 100, 1.8347, 24, 666, 20.199999999999999, 27.25, 29.050000000000001, 7.2000000000000002);
+INSERT INTO boston VALUES (420, 11.8123, 0, 18.100000000000001, 0, 0.71799999999999997, 6.8239999999999998, 76.5, 1.794, 24, 666, 20.199999999999999, 48.450000000000003, 22.739999999999998, 8.4000000000000004);
+INSERT INTO boston VALUES (424, 7.0504199999999999, 0, 18.100000000000001, 0, 0.61399999999999999, 6.1029999999999998, 85.099999999999994, 2.0217999999999998, 24, 666, 20.199999999999999, 2.52, 23.289999999999999, 13.4);
+INSERT INTO boston VALUES (428, 37.661900000000003, 0, 18.100000000000001, 0, 0.67900000000000005, 6.202, 78.700000000000003, 1.8629, 24, 666, 20.199999999999999, 18.82, 14.52, 10.9);
+INSERT INTO boston VALUES (432, 10.0623, 0, 18.100000000000001, 0, 0.58399999999999996, 6.8330000000000002, 94.299999999999997, 2.0882000000000001, 24, 666, 20.199999999999999, 81.329999999999998, 19.690000000000001, 14.1);
+INSERT INTO boston VALUES (436, 11.160399999999999, 0, 18.100000000000001, 0, 0.73999999999999999, 6.6289999999999996, 94.599999999999994, 2.1246999999999998, 24, 666, 20.199999999999999, 109.84999999999999, 23.27, 13.4);
+INSERT INTO boston VALUES (440, 9.3906299999999998, 0, 18.100000000000001, 0, 0.73999999999999999, 5.6269999999999998, 93.900000000000006, 1.8171999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 22.879999999999999, 12.800000000000001);
+INSERT INTO boston VALUES (444, 9.9665400000000002, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4850000000000003, 100, 1.9783999999999999, 24, 666, 20.199999999999999, 386.73000000000002, 18.850000000000001, 15.4);
+INSERT INTO boston VALUES (448, 9.9248499999999993, 0, 18.100000000000001, 0, 0.73999999999999999, 6.2510000000000003, 96.599999999999994, 2.198, 24, 666, 20.199999999999999, 388.51999999999998, 16.440000000000001, 12.6);
+INSERT INTO boston VALUES (452, 5.4411399999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.6550000000000002, 98.200000000000003, 2.3552, 24, 666, 20.199999999999999, 355.29000000000002, 17.73, 15.199999999999999);
+INSERT INTO boston VALUES (456, 4.75237, 0, 18.100000000000001, 0, 0.71299999999999997, 6.5250000000000004, 86.5, 2.4358, 24, 666, 20.199999999999999, 50.920000000000002, 18.129999999999999, 14.1);
+INSERT INTO boston VALUES (460, 6.8011699999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.0810000000000004, 84.400000000000006, 2.7174999999999998, 24, 666, 20.199999999999999, 396.89999999999998, 14.699999999999999, 20);
+INSERT INTO boston VALUES (236, 0.33045000000000002, 0, 6.2000000000000002, 0, 0.50700000000000001, 6.0860000000000003, 61.5, 3.6518999999999999, 8, 307, 17.399999999999999, 376.75, 10.880000000000001, 24);
+INSERT INTO boston VALUES (240, 0.092520000000000005, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.6059999999999999, 42.200000000000003, 6.1898999999999997, 6, 300, 16.600000000000001, 383.77999999999997, 7.3700000000000001, 23.300000000000001);
+INSERT INTO boston VALUES (244, 0.12756999999999999, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.3929999999999998, 7.7999999999999998, 7.0354999999999999, 6, 300, 16.600000000000001, 374.70999999999998, 5.1900000000000004, 23.699999999999999);
+INSERT INTO boston VALUES (248, 0.19656999999999999, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.226, 79.200000000000003, 8.0555000000000003, 7, 330, 19.100000000000001, 376.13999999999999, 10.15, 20.5);
+INSERT INTO boston VALUES (252, 0.21409, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.4379999999999997, 8.9000000000000004, 7.3967000000000001, 7, 330, 19.100000000000001, 377.06999999999999, 3.5899999999999999, 24.800000000000001);
+INSERT INTO boston VALUES (259, 0.66351000000000004, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.3330000000000002, 100, 1.8946000000000001, 5, 264, 13, 383.29000000000002, 7.79, 36);
+INSERT INTO boston VALUES (263, 0.52014000000000005, 20, 3.9700000000000002, 0, 0.64700000000000002, 8.3979999999999997, 91.5, 2.2885, 5, 264, 13, 386.86000000000001, 5.9100000000000001, 48.799999999999997);
+INSERT INTO boston VALUES (267, 0.78569999999999995, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.0140000000000002, 84.599999999999994, 2.1328999999999998, 5, 264, 13, 384.06999999999999, 14.789999999999999, 30.699999999999999);
+INSERT INTO boston VALUES (271, 0.29915999999999998, 20, 6.96, 0, 0.46400000000000002, 5.8559999999999999, 42.100000000000001, 4.4290000000000003, 3, 223, 18.600000000000001, 388.64999999999998, 13, 21.100000000000001);
+INSERT INTO boston VALUES (275, 0.056439999999999997, 40, 6.4100000000000001, 1, 0.44700000000000001, 6.758, 32.899999999999999, 4.0776000000000003, 4, 254, 17.600000000000001, 396.89999999999998, 3.5299999999999998, 32.399999999999999);
+INSERT INTO boston VALUES (279, 0.079780000000000004, 40, 6.4100000000000001, 0, 0.44700000000000001, 6.4820000000000002, 32.100000000000001, 4.1402999999999999, 4, 254, 17.600000000000001, 396.89999999999998, 7.1900000000000004, 29.100000000000001);
+INSERT INTO boston VALUES (283, 0.061289999999999997, 20, 3.3300000000000001, 1, 0.44290000000000002, 7.6449999999999996, 49.700000000000003, 5.2119, 5, 216, 14.9, 377.06999999999999, 3.0099999999999998, 46);
+INSERT INTO boston VALUES (287, 0.019650000000000001, 80, 1.76, 0, 0.38500000000000001, 6.2300000000000004, 31.5, 9.0891999999999999, 1, 241, 18.199999999999999, 341.60000000000002, 12.93, 20.100000000000001);
+INSERT INTO boston VALUES (291, 0.035020000000000003, 80, 4.9500000000000002, 0, 0.41099999999999998, 6.8609999999999998, 27.899999999999999, 5.1166999999999998, 4, 245, 19.199999999999999, 396.89999999999998, 3.3300000000000001, 28.5);
+INSERT INTO boston VALUES (295, 0.081989999999999993, 0, 13.92, 0, 0.437, 6.0090000000000003, 42.299999999999997, 5.5026999999999999, 4, 289, 16, 396.89999999999998, 10.4, 21.699999999999999);
+INSERT INTO boston VALUES (299, 0.064659999999999995, 70, 2.2400000000000002, 0, 0.40000000000000002, 6.3449999999999998, 20.100000000000001, 7.8277999999999999, 5, 358, 14.800000000000001, 368.24000000000001, 4.9699999999999998, 22.5);
+INSERT INTO boston VALUES (303, 0.092660000000000006, 34, 6.0899999999999999, 0, 0.433, 6.4950000000000001, 18.399999999999999, 5.4916999999999998, 7, 329, 16.100000000000001, 383.61000000000001, 8.6699999999999999, 26.399999999999999);
+INSERT INTO boston VALUES (307, 0.075029999999999999, 33, 2.1800000000000002, 0, 0.47199999999999998, 7.4199999999999999, 71.900000000000006, 3.0992000000000002, 7, 222, 18.399999999999999, 396.89999999999998, 6.4699999999999998, 33.399999999999999);
+INSERT INTO boston VALUES (311, 2.6354799999999998, 0, 9.9000000000000004, 0, 0.54400000000000004, 4.9729999999999999, 37.799999999999997, 2.5194000000000001, 4, 304, 18.399999999999999, 350.44999999999999, 12.640000000000001, 16.100000000000001);
+INSERT INTO boston VALUES (315, 0.36919999999999997, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.5670000000000002, 87.299999999999997, 3.6023000000000001, 4, 304, 18.399999999999999, 395.69, 9.2799999999999994, 23.800000000000001);
+INSERT INTO boston VALUES (319, 0.40201999999999999, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.3819999999999997, 67.200000000000003, 3.5325000000000002, 4, 304, 18.399999999999999, 395.20999999999998, 10.359999999999999, 23.100000000000001);
+INSERT INTO boston VALUES (323, 0.35114000000000001, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.0410000000000004, 49.899999999999999, 4.7210999999999999, 5, 287, 19.600000000000001, 396.89999999999998, 7.7000000000000002, 20.399999999999999);
+INSERT INTO boston VALUES (327, 0.30347000000000002, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.3120000000000003, 28.899999999999999, 5.4158999999999997, 5, 287, 19.600000000000001, 396.89999999999998, 6.1500000000000004, 23);
+INSERT INTO boston VALUES (331, 0.045440000000000001, 0, 3.2400000000000002, 0, 0.46000000000000002, 6.1440000000000001, 32.200000000000003, 5.8735999999999997, 4, 430, 16.899999999999999, 368.56999999999999, 9.0899999999999999, 19.800000000000001);
+INSERT INTO boston VALUES (335, 0.037379999999999997, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.3099999999999996, 38.5, 6.4584000000000001, 5, 224, 20.199999999999999, 389.39999999999998, 6.75, 20.699999999999999);
+INSERT INTO boston VALUES (339, 0.033059999999999999, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.0590000000000002, 37.299999999999997, 4.8121999999999998, 5, 224, 20.199999999999999, 396.13999999999999, 8.5099999999999998, 20.600000000000001);
+INSERT INTO boston VALUES (343, 0.024979999999999999, 0, 1.8899999999999999, 0, 0.51800000000000002, 6.54, 59.700000000000003, 6.2668999999999997, 1, 422, 15.9, 389.95999999999998, 8.6500000000000004, 16.5);
+INSERT INTO boston VALUES (347, 0.061620000000000001, 0, 4.3899999999999997, 0, 0.442, 5.8979999999999997, 52.299999999999997, 8.0136000000000003, 3, 352, 18.800000000000001, 364.61000000000001, 12.67, 17.199999999999999);
+INSERT INTO boston VALUES (351, 0.062109999999999999, 40, 1.25, 0, 0.42899999999999999, 6.4900000000000002, 44.399999999999999, 8.7920999999999996, 1, 335, 19.699999999999999, 396.89999999999998, 5.9800000000000004, 22.899999999999999);
+INSERT INTO boston VALUES (355, 0.04301, 80, 1.9099999999999999, 0, 0.41299999999999998, 5.6630000000000003, 21.899999999999999, 10.585699999999999, 4, 334, 22, 382.80000000000001, 8.0500000000000007, 18.199999999999999);
+INSERT INTO boston VALUES (359, 5.2017699999999998, 0, 18.100000000000001, 1, 0.77000000000000002, 6.1269999999999998, 83.400000000000006, 2.7227000000000001, 24, 666, 20.199999999999999, 395.43000000000001, 11.48, 22.699999999999999);
+INSERT INTO boston VALUES (363, 3.67822, 0, 18.100000000000001, 0, 0.77000000000000002, 5.3620000000000001, 96.200000000000003, 2.1036000000000001, 24, 666, 20.199999999999999, 380.79000000000002, 10.19, 20.800000000000001);
+INSERT INTO boston VALUES (367, 3.6969500000000002, 0, 18.100000000000001, 0, 0.71799999999999997, 4.9630000000000001, 91.400000000000006, 1.7523, 24, 666, 20.199999999999999, 316.02999999999997, 14, 21.899999999999999);
+INSERT INTO boston VALUES (371, 6.5387599999999999, 0, 18.100000000000001, 1, 0.63100000000000001, 7.016, 97.5, 1.2023999999999999, 24, 666, 20.199999999999999, 392.05000000000001, 2.96, 50);
+INSERT INTO boston VALUES (375, 18.498200000000001, 0, 18.100000000000001, 0, 0.66800000000000004, 4.1379999999999999, 100, 1.137, 24, 666, 20.199999999999999, 396.89999999999998, 37.969999999999999, 13.800000000000001);
+INSERT INTO boston VALUES (379, 23.648199999999999, 0, 18.100000000000001, 0, 0.67100000000000004, 6.3799999999999999, 96.200000000000003, 1.3861000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 23.690000000000001, 13.1);
+INSERT INTO boston VALUES (383, 9.1870200000000004, 0, 18.100000000000001, 0, 0.69999999999999996, 5.5359999999999996, 100, 1.5804, 24, 666, 20.199999999999999, 396.89999999999998, 23.600000000000001, 11.300000000000001);
+INSERT INTO boston VALUES (387, 24.393799999999999, 0, 18.100000000000001, 0, 0.69999999999999996, 4.6520000000000001, 100, 1.4672000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 28.280000000000001, 10.5);
+INSERT INTO boston VALUES (391, 6.9621500000000003, 0, 18.100000000000001, 0, 0.69999999999999996, 5.7130000000000001, 97, 1.9265000000000001, 24, 666, 20.199999999999999, 394.43000000000001, 17.109999999999999, 15.1);
+INSERT INTO boston VALUES (395, 13.3598, 0, 18.100000000000001, 0, 0.69299999999999995, 5.8869999999999996, 94.700000000000003, 1.7821, 24, 666, 20.199999999999999, 396.89999999999998, 16.350000000000001, 12.699999999999999);
+INSERT INTO boston VALUES (399, 38.351799999999997, 0, 18.100000000000001, 0, 0.69299999999999995, 5.4530000000000003, 100, 1.4896, 24, 666, 20.199999999999999, 396.89999999999998, 30.59, 5);
+INSERT INTO boston VALUES (403, 9.5957100000000004, 0, 18.100000000000001, 0, 0.69299999999999995, 6.4039999999999999, 100, 1.639, 24, 666, 20.199999999999999, 376.11000000000001, 20.309999999999999, 12.1);
+INSERT INTO boston VALUES (407, 20.716200000000001, 0, 18.100000000000001, 0, 0.65900000000000003, 4.1379999999999999, 100, 1.1780999999999999, 24, 666, 20.199999999999999, 370.22000000000003, 23.34, 11.9);
+INSERT INTO boston VALUES (411, 51.135800000000003, 0, 18.100000000000001, 0, 0.59699999999999998, 5.7569999999999997, 100, 1.413, 24, 666, 20.199999999999999, 2.6000000000000001, 10.109999999999999, 15);
+INSERT INTO boston VALUES (415, 45.746099999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 4.5190000000000001, 100, 1.6581999999999999, 24, 666, 20.199999999999999, 88.269999999999996, 36.979999999999997, 7);
+INSERT INTO boston VALUES (419, 73.534099999999995, 0, 18.100000000000001, 0, 0.67900000000000005, 5.9569999999999999, 100, 1.8026, 24, 666, 20.199999999999999, 16.449999999999999, 20.620000000000001, 8.8000000000000007);
+INSERT INTO boston VALUES (423, 12.0482, 0, 18.100000000000001, 0, 0.61399999999999999, 5.6479999999999997, 87.599999999999994, 1.9512, 24, 666, 20.199999999999999, 291.55000000000001, 14.1, 20.800000000000001);
+INSERT INTO boston VALUES (427, 12.247199999999999, 0, 18.100000000000001, 0, 0.58399999999999996, 5.8369999999999997, 59.700000000000003, 1.9976, 24, 666, 20.199999999999999, 24.649999999999999, 15.69, 10.199999999999999);
+INSERT INTO boston VALUES (431, 8.4921299999999995, 0, 18.100000000000001, 0, 0.58399999999999996, 6.3479999999999999, 86.099999999999994, 2.0527000000000002, 24, 666, 20.199999999999999, 83.450000000000003, 17.640000000000001, 14.5);
+INSERT INTO boston VALUES (435, 13.913399999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.2080000000000002, 95, 2.2222, 24, 666, 20.199999999999999, 100.63, 15.17, 11.699999999999999);
+INSERT INTO boston VALUES (439, 13.678100000000001, 0, 18.100000000000001, 0, 0.73999999999999999, 5.9349999999999996, 87.900000000000006, 1.8206, 24, 666, 20.199999999999999, 68.950000000000003, 34.020000000000003, 8.4000000000000004);
+INSERT INTO boston VALUES (443, 5.6663699999999997, 0, 18.100000000000001, 0, 0.73999999999999999, 6.2190000000000003, 100, 2.0047999999999999, 24, 666, 20.199999999999999, 395.69, 16.59, 18.399999999999999);
+INSERT INTO boston VALUES (447, 6.2880700000000003, 0, 18.100000000000001, 0, 0.73999999999999999, 6.3410000000000002, 96.400000000000006, 2.0720000000000001, 24, 666, 20.199999999999999, 318.00999999999999, 17.789999999999999, 14.9);
+INSERT INTO boston VALUES (451, 6.7177199999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.7489999999999997, 92.599999999999994, 2.3235999999999999, 24, 666, 20.199999999999999, 0.32000000000000001, 17.440000000000001, 13.4);
+INSERT INTO boston VALUES (455, 9.5136299999999991, 0, 18.100000000000001, 0, 0.71299999999999997, 6.7279999999999998, 94.099999999999994, 2.4961000000000002, 24, 666, 20.199999999999999, 6.6799999999999997, 18.710000000000001, 14.9);
+INSERT INTO boston VALUES (459, 7.75223, 0, 18.100000000000001, 0, 0.71299999999999997, 6.3010000000000002, 83.700000000000003, 2.7831000000000001, 24, 666, 20.199999999999999, 272.20999999999998, 16.23, 14.9);
+INSERT INTO boston VALUES (463, 6.6549199999999997, 0, 18.100000000000001, 0, 0.71299999999999997, 6.3170000000000002, 83, 2.7343999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 13.99, 19.5);
+INSERT INTO boston VALUES (467, 3.7749799999999998, 0, 18.100000000000001, 0, 0.65500000000000003, 5.952, 84.700000000000003, 2.8715000000000002, 24, 666, 20.199999999999999, 22.010000000000002, 17.149999999999999, 19);
+INSERT INTO boston VALUES (466, 3.1636000000000002, 0, 18.100000000000001, 0, 0.65500000000000003, 5.7590000000000003, 48.200000000000003, 3.0665, 24, 666, 20.199999999999999, 334.39999999999998, 14.130000000000001, 19.899999999999999);
+INSERT INTO boston VALUES (470, 13.075100000000001, 0, 18.100000000000001, 0, 0.57999999999999996, 5.7130000000000001, 56.700000000000003, 2.8237000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 14.76, 20.100000000000001);
+INSERT INTO boston VALUES (474, 4.64689, 0, 18.100000000000001, 0, 0.61399999999999999, 6.9800000000000004, 67.599999999999994, 2.5329000000000002, 24, 666, 20.199999999999999, 374.68000000000001, 11.66, 29.800000000000001);
+INSERT INTO boston VALUES (478, 15.023400000000001, 0, 18.100000000000001, 0, 0.61399999999999999, 5.3040000000000003, 97.299999999999997, 2.1006999999999998, 24, 666, 20.199999999999999, 349.48000000000002, 24.91, 12);
+INSERT INTO boston VALUES (482, 5.7081799999999996, 0, 18.100000000000001, 0, 0.53200000000000003, 6.75, 74.900000000000006, 3.3317000000000001, 24, 666, 20.199999999999999, 393.06999999999999, 7.7400000000000002, 23.699999999999999);
+INSERT INTO boston VALUES (486, 3.67367, 0, 18.100000000000001, 0, 0.58299999999999996, 6.3120000000000003, 51.899999999999999, 3.9916999999999998, 24, 666, 20.199999999999999, 388.62, 10.58, 21.199999999999999);
+INSERT INTO boston VALUES (490, 0.18337000000000001, 0, 27.739999999999998, 0, 0.60899999999999999, 5.4139999999999997, 98.299999999999997, 1.7554000000000001, 4, 711, 20.100000000000001, 344.05000000000001, 23.969999999999999, 7);
+INSERT INTO boston VALUES (494, 0.17330999999999999, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.7069999999999999, 54, 2.3816999999999999, 6, 391, 19.199999999999999, 396.89999999999998, 12.01, 21.800000000000001);
+INSERT INTO boston VALUES (498, 0.26838000000000001, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.7939999999999996, 70.599999999999994, 2.8927, 6, 391, 19.199999999999999, 396.89999999999998, 14.1, 18.300000000000001);
+INSERT INTO boston VALUES (502, 0.062630000000000005, 0, 11.93, 0, 0.57299999999999995, 6.593, 69.099999999999994, 2.4786000000000001, 1, 273, 21, 391.99000000000001, 9.6699999999999999, 22.399999999999999);
+INSERT INTO boston VALUES (506, 0.047410000000000001, 0, 11.93, 0, 0.57299999999999995, 6.0300000000000002, 80.799999999999997, 2.5049999999999999, 1, 273, 21, 396.89999999999998, 7.8799999999999999, 11.9);
+INSERT INTO boston VALUES (464, 5.8211500000000003, 0, 18.100000000000001, 0, 0.71299999999999997, 6.5129999999999999, 89.900000000000006, 2.8016000000000001, 24, 666, 20.199999999999999, 393.81999999999999, 10.289999999999999, 20.199999999999999);
+INSERT INTO boston VALUES (468, 4.4222799999999998, 0, 18.100000000000001, 0, 0.58399999999999996, 6.0030000000000001, 94.5, 2.5402999999999998, 24, 666, 20.199999999999999, 331.29000000000002, 21.32, 19.100000000000001);
+INSERT INTO boston VALUES (472, 4.0384099999999998, 0, 18.100000000000001, 0, 0.53200000000000003, 6.2290000000000001, 90.700000000000003, 3.0992999999999999, 24, 666, 20.199999999999999, 395.32999999999998, 12.869999999999999, 19.600000000000001);
+INSERT INTO boston VALUES (476, 6.3931199999999997, 0, 18.100000000000001, 0, 0.58399999999999996, 6.1619999999999999, 97.400000000000006, 2.206, 24, 666, 20.199999999999999, 302.75999999999999, 24.100000000000001, 13.300000000000001);
+INSERT INTO boston VALUES (480, 14.3337, 0, 18.100000000000001, 0, 0.61399999999999999, 6.2290000000000001, 88, 1.9512, 24, 666, 20.199999999999999, 383.31999999999999, 13.109999999999999, 21.399999999999999);
+INSERT INTO boston VALUES (484, 2.8183799999999999, 0, 18.100000000000001, 0, 0.53200000000000003, 5.7619999999999996, 40.299999999999997, 4.0983000000000001, 24, 666, 20.199999999999999, 392.92000000000002, 10.42, 21.800000000000001);
+INSERT INTO boston VALUES (488, 4.8356700000000004, 0, 18.100000000000001, 0, 0.58299999999999996, 5.9050000000000002, 53.200000000000003, 3.1522999999999999, 24, 666, 20.199999999999999, 388.22000000000003, 11.449999999999999, 20.600000000000001);
+INSERT INTO boston VALUES (492, 0.10574, 0, 27.739999999999998, 0, 0.60899999999999999, 5.9829999999999997, 98.799999999999997, 1.8681000000000001, 4, 711, 20.100000000000001, 390.11000000000001, 18.07, 13.6);
+INSERT INTO boston VALUES (496, 0.17899000000000001, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.6699999999999999, 28.800000000000001, 2.7986, 6, 391, 19.199999999999999, 393.29000000000002, 17.600000000000001, 23.100000000000001);
+INSERT INTO boston VALUES (500, 0.17782999999999999, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.569, 73.5, 2.3999000000000001, 6, 391, 19.199999999999999, 395.76999999999998, 15.1, 17.5);
+INSERT INTO boston VALUES (504, 0.060760000000000002, 0, 11.93, 0, 0.57299999999999995, 6.976, 91, 2.1675, 1, 273, 21, 396.89999999999998, 5.6399999999999997, 23.899999999999999);
+INSERT INTO boston VALUES (471, 4.3487900000000002, 0, 18.100000000000001, 0, 0.57999999999999996, 6.1669999999999998, 84, 3.0333999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 16.289999999999999, 19.899999999999999);
+INSERT INTO boston VALUES (475, 8.05579, 0, 18.100000000000001, 0, 0.58399999999999996, 5.4269999999999996, 95.400000000000006, 2.4298000000000002, 24, 666, 20.199999999999999, 352.57999999999998, 18.140000000000001, 13.800000000000001);
+INSERT INTO boston VALUES (479, 10.233000000000001, 0, 18.100000000000001, 0, 0.61399999999999999, 6.1849999999999996, 96.700000000000003, 2.1705000000000001, 24, 666, 20.199999999999999, 379.69999999999999, 18.030000000000001, 14.6);
+INSERT INTO boston VALUES (483, 5.73116, 0, 18.100000000000001, 0, 0.53200000000000003, 7.0609999999999999, 77, 3.4106000000000001, 24, 666, 20.199999999999999, 395.27999999999997, 7.0099999999999998, 25);
+INSERT INTO boston VALUES (487, 5.6917499999999999, 0, 18.100000000000001, 0, 0.58299999999999996, 6.1139999999999999, 79.799999999999997, 3.5459000000000001, 24, 666, 20.199999999999999, 392.68000000000001, 14.98, 19.100000000000001);
+INSERT INTO boston VALUES (491, 0.20746000000000001, 0, 27.739999999999998, 0, 0.60899999999999999, 5.093, 98, 1.8226, 4, 711, 20.100000000000001, 318.43000000000001, 29.68, 8.0999999999999996);
+INSERT INTO boston VALUES (495, 0.27956999999999999, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.9260000000000002, 42.600000000000001, 2.3816999999999999, 6, 391, 19.199999999999999, 396.89999999999998, 13.59, 24.5);
+INSERT INTO boston VALUES (499, 0.23912, 0, 9.6899999999999995, 0, 0.58499999999999996, 6.0190000000000001, 65.299999999999997, 2.4091, 6, 391, 19.199999999999999, 396.89999999999998, 12.92, 21.199999999999999);
+INSERT INTO boston VALUES (503, 0.045269999999999998, 0, 11.93, 0, 0.57299999999999995, 6.1200000000000001, 76.700000000000003, 2.2875000000000001, 1, 273, 21, 396.89999999999998, 9.0800000000000001, 20.600000000000001);
+
+
+-------------- data insert finish -------------
+
+select MADLIB_SCHEMA.xgboost_sk_Regressor('boston', 'boston_model_xgbr', 'id', 'target', 'crim, zn, Indus, chas, nox, rm, "Age"', NULL, $${'booster': ['gbtree'], 'eta': (0.1), 'max_depth': (5,6,7), 'objective':['reg:linear']}$$, TRUE, TRUE);
+
+select count(*) from boston_model_xgbr;
+select id, train_timestamp, source_table, y_type, metrics, features, params from boston_model_xgbr;
+
+SELECT assert(count(*) = 3, 'xgboost: Xgboost model output has incorrect number of rows')
+FROM (select * from boston_model_xgbr) q;
+
+drop table boston_model_xgbr;
+select MADLIB_SCHEMA.xgboost_sk_Regressor('boston', 'boston_model_xgbr', 'id', 'target', 'crim, zn, Indus, chas, nox, rm, "Age"', NULL,
+ $${'booster': ['gbtree'], 'eta': (0.1), 'max_depth': (5,6,7), 'objective':['reg:linear']}$$);
+
+select id, train_timestamp, source_table, y_type, metrics, features, params from boston_model_xgbr;
+select assert(metrics = 'evalution is OFF, no metrics', 'xgboost metrics error') from boston_model_xgbr;
+
+drop table if exists boston_model_xgbr3;
+create table boston_model_xgbr3 as select * from boston_model_xgbr where id=3;
+
+select MADLIB_SCHEMA.xgboost_sk_predict('boston', 'boston_model_xgbr', 'boston_output_xgbr', 'id');
+
+drop table if exists boston_xgb_results;
+create table boston_xgb_results as select t1.id, prediction, target from boston_output_xgbr as t1, boston as t2 order by t1.id;
+
+select mean_squared_error('boston_xgb_results', 'table_out','prediction', 'target');
+
+select * from table_out;
\ No newline at end of file
diff --git a/src/ports/postgres/modules/xgboost_gs/test/xgboost_test2.sql_in b/src/ports/postgres/modules/xgboost_gs/test/xgboost_test2.sql_in
new file mode 100644
index 0000000..b3e3a09
--- /dev/null
+++ b/src/ports/postgres/modules/xgboost_gs/test/xgboost_test2.sql_in
@@ -0,0 +1,464 @@
+
+DROP TABLE IF EXISTS telecom;
+CREATE TABLE telecom(
+"customerID" serial, gender VARCHAR(8), "SeniorCitizen" BOOLEAN, "Partner" BOOLEAN, "Dependents" BOOLEAN,
+tenure INT, "PhoneService" VARCHAR(20), "MultipleLines" VARCHAR(20), "InternetService" VARCHAR(20), "OnlineSecurity" VARCHAR(20),
+"OnlineBackup" VARCHAR(20), "DeviceProtection" VARCHAR(20), "TechSupport" VARCHAR(20), "StreamingTV" VARCHAR(20), "StreamingMovies" VARCHAR(20),
+"Contract" VARCHAR(30), "PaperlessBilling" BOOLEAN, "PaymentMethod" VARCHAR(40), "MonthlyCharges" FLOAT DEFAULT 0, "TotalCharges" FLOAT DEFAULT 0, "Churn" INT DEFAULT 0);
+
+COPY telecom (gender, "SeniorCitizen", "Partner", "Dependents", tenure, "PhoneService", "MultipleLines", "InternetService", "OnlineSecurity", "OnlineBackup", "DeviceProtection", "TechSupport", "StreamingTV", "StreamingMovies", "Contract", "PaperlessBilling", "PaymentMethod", "MonthlyCharges", "TotalCharges", "Churn") FROM STDIN DELIMITER ',';
+Male,0,0,0,2,1,0,DSL,1,1,0,0,0,0,Month-to-month,1,Mailed check,53.85,108.15,1
+Male,0,0,0,45,0,0 phone service,DSL,1,0,1,1,0,0,One year,0,Bank transfer (automatic),42.3,1840.75,0
+Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,70.7,151.65,1
+Female,0,0,0,8,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,99.65,820.5,1
+Male,0,0,1,22,1,1,Fiber optic,0,1,0,0,1,0,Month-to-month,1,Credit card (automatic),89.1,1949.4,0
+Female,0,0,0,10,0,0 phone service,DSL,1,0,0,0,0,0,Month-to-month,0,Mailed check,29.75,301.9,0
+Female,0,1,0,28,1,1,Fiber optic,0,0,1,1,1,1,Month-to-month,1,Electronic check,104.8,3046.05,1
+Male,0,0,1,62,1,0,DSL,1,1,0,0,0,0,One year,0,Bank transfer (automatic),56.15,3487.95,0
+Male,0,1,1,13,1,0,DSL,1,0,0,0,0,0,Month-to-month,1,Mailed check,49.95,587.45,0
+Male,0,0,0,16,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),18.95,326.8,0
+Male,0,1,0,58,1,1,Fiber optic,0,0,1,0,1,1,One year,0,Credit card (automatic),100.35,5681.1,0
+Male,0,0,0,49,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Bank transfer (automatic),103.7,5036.3,1
+Male,0,0,0,25,1,0,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Electronic check,105.5,2686.05,0
+Female,0,1,1,69,1,1,Fiber optic,1,1,1,1,1,1,Two year,0,Credit card (automatic),113.25,7895.15,0
+Female,0,0,0,52,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.65,1022.95,0
+Male,0,0,1,71,1,1,Fiber optic,1,0,1,0,1,1,Two year,0,Bank transfer (automatic),106.7,7382.25,0
+Female,0,1,1,10,1,0,DSL,0,0,1,1,0,0,Month-to-month,0,Credit card (automatic),55.2,528.35,1
+Female,0,0,0,21,1,0,Fiber optic,0,1,1,0,0,1,Month-to-month,1,Electronic check,90.05,1862.9,0
+Male,1,0,0,1,0,0 phone service,DSL,0,0,1,0,0,1,Month-to-month,1,Electronic check,39.65,39.65,1
+Male,0,1,0,12,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),19.8,202.25,0
+Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.15,20.15,1
+Female,0,1,0,58,1,1,DSL,0,1,0,1,0,0,Two year,1,Credit card (automatic),59.9,3505.1,0
+Male,0,1,1,49,1,0,DSL,1,1,0,1,0,0,Month-to-month,0,Credit card (automatic),59.6,2970.3,0
+Female,0,0,0,30,1,0,DSL,1,1,0,0,0,0,Month-to-month,1,Bank transfer (automatic),55.3,1530.6,0
+Male,0,1,1,47,1,1,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Electronic check,99.35,4749.15,1
+Male,0,1,1,1,0,0 phone service,DSL,0,1,0,0,0,0,Month-to-month,0,Electronic check,30.2,30.2,1
+Male,0,1,0,72,1,1,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),90.25,6369.45,0
+Female,0,0,1,17,1,0,DSL,0,0,0,0,1,1,Month-to-month,1,Mailed check,64.7,1093.1,1
+Female,1,1,0,71,1,1,Fiber optic,1,1,1,1,0,0,Two year,1,Credit card (automatic),96.35,6766.95,0
+Male,1,1,0,2,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Credit card (automatic),95.5,181.65,0
+Female,0,1,1,27,1,0,DSL,1,1,1,1,0,0,One year,0,Mailed check,66.15,1874.45,0
+Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Bank transfer (automatic),20.2,20.2,0
+Male,1,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),45.25,45.25,0
+Female,0,1,1,72,1,1,Fiber optic,1,1,0,1,1,0,Two year,0,Bank transfer (automatic),99.9,7251.7,0
+Male,0,0,0,5,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,69.7,316.9,1
+Female,0,0,0,46,1,0,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Credit card (automatic),74.8,3548.3,0
+Male,0,0,0,34,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,106.35,3549.25,1
+Female,0,0,0,11,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Bank transfer (automatic),97.85,1105.4,1
+Male,0,1,1,10,1,0,DSL,0,1,0,0,0,0,One year,0,Mailed check,49.55,475.7,0
+Female,0,1,1,70,1,1,DSL,1,1,0,0,1,0,Two year,1,Credit card (automatic),69.2,4872.35,0
+Female,0,1,1,17,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.75,418.25,0
+Female,0,0,0,63,1,1,DSL,1,1,1,1,1,0,Two year,1,Credit card (automatic),79.85,4861.45,0
+Female,0,1,0,13,1,1,DSL,1,1,0,1,1,0,Month-to-month,1,Electronic check,76.2,981.45,0
+Female,0,0,0,49,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Electronic check,84.5,3906.7,0
+Male,0,0,0,2,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Mailed check,49.25,97,0
+Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,80.65,144.15,1
+Male,0,0,0,52,1,1,DSL,1,0,0,1,1,1,Two year,1,Credit card (automatic),79.75,4217.8,0
+Female,0,1,1,69,1,1,DSL,1,0,1,1,0,0,Two year,1,Credit card (automatic),64.15,4254.1,0
+Female,1,0,0,43,1,1,Fiber optic,0,1,0,0,1,0,Month-to-month,1,Electronic check,90.25,3838.75,0
+Female,0,0,0,15,1,0,Fiber optic,1,1,0,0,1,1,Month-to-month,1,Credit card (automatic),99.1,1426.4,1
+Female,1,1,0,25,1,1,DSL,1,0,0,1,1,0,Month-to-month,1,Credit card (automatic),69.5,1752.65,0
+Female,1,1,0,8,1,1,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Credit card (automatic),80.65,633.3,1
+Female,1,1,1,60,1,0,DSL,1,1,1,1,0,1,One year,1,Credit card (automatic),74.85,4456.35,0
+Male,1,0,0,18,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,95.45,1752.55,1
+Female,0,1,1,63,1,1,Fiber optic,1,0,0,0,1,1,One year,1,Credit card (automatic),99.65,6311.2,0
+Male,1,1,1,66,1,1,Fiber optic,0,1,1,1,1,1,One year,1,Electronic check,108.45,7076.35,0
+Female,0,1,1,34,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),24.95,894.3,0
+Female,0,0,0,72,1,1,Fiber optic,0,0,1,1,1,1,Two year,1,Credit card (automatic),107.5,7853.7,0
+Female,0,1,0,47,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,100.5,4707.1,0
+Male,0,0,0,60,1,1,Fiber optic,0,1,0,0,1,0,Month-to-month,1,Bank transfer (automatic),89.9,5450.7,0
+Male,0,1,0,72,0,0 phone service,DSL,1,1,1,0,0,0,Two year,0,Bank transfer (automatic),42.1,2962,0
+Female,0,1,1,18,1,0,DSL,0,0,1,1,0,0,One year,1,Credit card (automatic),54.4,957.1,0
+Female,0,0,0,9,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,0,Electronic check,94.4,857.25,1
+Female,0,0,0,3,1,0,DSL,0,1,0,1,1,1,Month-to-month,1,Electronic check,75.3,244.1,0
+Male,0,1,0,47,1,1,Fiber optic,0,1,0,0,0,0,One year,1,Electronic check,78.9,3650.35,0
+Female,0,0,0,31,1,0,DSL,0,1,1,1,1,1,Two year,0,Mailed check,79.2,2497.2,0
+Female,0,1,1,50,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),20.15,930.9,0
+Male,0,0,0,10,1,0,Fiber optic,1,0,1,0,0,0,Month-to-month,1,Mailed check,79.85,887.35,0
+Male,0,0,0,1,1,0,DSL,0,0,0,1,0,0,Month-to-month,0,Mailed check,49.05,49.05,0
+Female,0,1,1,52,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Electronic check,20.4,1090.65,0
+Male,1,1,1,64,1,1,Fiber optic,1,0,1,1,1,1,Two year,1,Bank transfer (automatic),111.6,7099,0
+Male,0,1,1,62,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),24.25,1424.6,0
+Female,0,0,1,3,1,0,DSL,1,0,0,1,0,1,Month-to-month,1,Bank transfer (automatic),64.5,177.4,0
+Female,1,0,0,56,1,1,Fiber optic,1,1,1,0,1,1,One year,0,Electronic check,110.5,6139.5,0
+Female,0,0,0,46,1,0,DSL,0,0,0,0,0,1,One year,0,Credit card (automatic),55.65,2688.85,0
+Female,0,1,1,8,1,0,DSL,1,1,0,0,0,0,Month-to-month,0,Mailed check,54.65,482.25,0
+Male,1,0,0,30,1,0,DSL,1,1,0,0,1,1,Month-to-month,1,Electronic check,74.75,2111.3,0
+Female,0,1,1,45,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Credit card (automatic),25.9,1216.6,0
+Female,0,0,1,1,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,79.35,79.35,1
+Female,0,1,1,11,0,0 phone service,DSL,1,0,0,0,1,1,Month-to-month,0,Electronic check,50.55,565.35,0
+Female,0,1,0,7,1,0,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Bank transfer (automatic),75.15,496.9,1
+Female,0,0,0,42,1,0,Fiber optic,0,1,1,1,1,1,Month-to-month,1,Bank transfer (automatic),103.8,4327.5,0
+Female,0,1,0,49,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),20.15,973.35,0
+Male,0,0,0,9,1,1,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Credit card (automatic),99.3,918.75,0
+Female,0,1,0,35,1,0,DSL,1,0,0,0,1,0,One year,1,Bank transfer (automatic),62.15,2215.45,0
+Female,0,1,1,48,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),20.65,1057,0
+Female,0,1,1,46,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.95,927.1,0
+Male,0,1,0,29,0,0 phone service,DSL,0,0,0,0,1,0,Month-to-month,0,Mailed check,33.75,1009.25,0
+Male,0,1,1,30,1,0,Fiber optic,0,1,1,0,0,0,Month-to-month,0,Bank transfer (automatic),82.05,2570.2,0
+Male,1,0,0,1,1,0,Fiber optic,0,0,0,1,0,0,Month-to-month,0,Electronic check,74.7,74.7,0
+Male,0,1,1,66,1,1,DSL,1,0,1,1,1,1,Two year,1,Mailed check,84,5714.25,0
+Female,0,0,0,65,1,1,Fiber optic,1,1,1,0,1,1,Month-to-month,1,Credit card (automatic),111.05,7107,0
+Male,0,0,0,72,1,1,Fiber optic,0,1,1,0,1,1,Two year,1,Bank transfer (automatic),100.9,7459.05,0
+Female,0,0,0,12,1,1,Fiber optic,1,0,0,0,0,0,Month-to-month,1,Electronic check,78.95,927.35,1
+Male,0,1,1,71,1,1,DSL,1,1,0,1,0,0,One year,1,Credit card (automatic),66.85,4748.7,0
+Male,0,0,0,5,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,21.05,113.85,1
+Male,0,0,0,52,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),21,1107.2,0
+Female,1,1,0,25,1,0,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,98.5,2514.5,1
+Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.2,20.2,0
+Female,0,1,1,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,19.45,19.45,0
+Male,0,0,0,38,1,1,Fiber optic,0,0,1,1,1,0,One year,0,Bank transfer (automatic),95,3605.6,0
+Female,1,1,0,66,0,0 phone service,DSL,0,1,1,0,1,0,One year,0,Bank transfer (automatic),45.55,3027.25,0
+Male,0,1,0,68,1,1,Fiber optic,0,1,1,1,1,1,Two year,1,Bank transfer (automatic),110,7611.85,1
+Male,0,0,0,5,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Mailed check,24.3,100.2,0
+Female,0,1,1,72,1,1,Fiber optic,1,1,0,0,1,1,One year,1,Electronic check,104.15,7303.05,0
+Female,0,0,0,32,0,0 phone service,DSL,1,0,0,0,0,0,One year,0,Mailed check,30.15,927.65,0
+Male,0,0,0,43,1,1,Fiber optic,0,0,0,0,1,1,One year,0,Mailed check,94.35,3921.3,0
+Male,0,1,1,72,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),19.4,1363.25,0
+Male,0,1,0,55,1,1,Fiber optic,1,1,0,0,1,0,Month-to-month,1,Electronic check,96.75,5238.9,1
+Female,0,0,0,52,1,0,DSL,1,0,1,1,0,0,One year,0,Credit card (automatic),57.95,3042.25,0
+Female,0,0,0,43,1,0,Fiber optic,0,1,1,0,1,0,Month-to-month,1,Electronic check,91.65,3954.1,0
+Female,1,1,0,37,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,76.5,2868.15,1
+Female,0,1,1,64,0,0 phone service,DSL,0,1,0,1,1,1,Two year,1,Electronic check,54.6,3423.5,0
+Male,0,1,1,3,1,0,Fiber optic,1,1,0,0,1,0,Month-to-month,0,Electronic check,89.85,248.4,0
+Female,0,0,0,36,0,0 phone service,DSL,1,0,0,0,0,0,Two year,0,Bank transfer (automatic),31.05,1126.35,0
+Female,0,1,1,10,1,1,Fiber optic,1,0,0,0,1,1,Month-to-month,1,Electronic check,100.25,1064.65,1
+Female,0,0,0,41,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,20.65,835.15,0
+Male,0,1,1,27,1,1,Fiber optic,0,1,0,1,0,0,Month-to-month,1,Credit card (automatic),85.2,2151.6,0
+Female,0,1,1,56,1,1,Fiber optic,0,0,1,0,1,1,One year,1,Credit card (automatic),99.8,5515.45,0
+Female,0,0,0,6,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.7,112.75,0
+Male,0,0,0,3,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,74.4,229.55,1
+Female,0,1,1,7,1,0,DSL,1,0,0,0,0,0,Month-to-month,0,Mailed check,50.7,350.35,0
+Female,0,1,1,4,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.85,62.9,0
+Male,0,0,0,33,1,0,Fiber optic,1,0,0,1,1,0,Two year,1,Electronic check,88.95,3027.65,0
+Female,1,0,0,27,1,1,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Electronic check,78.05,2135.5,1
+Male,0,1,0,72,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),23.55,1723.95,0
+Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.75,19.75,0
+Male,1,0,0,71,0,0 phone service,DSL,1,1,0,0,1,1,One year,1,Electronic check,56.45,3985.35,0
+Female,0,0,0,13,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Electronic check,85.95,1215.65,0
+Female,0,1,1,25,0,0 phone service,DSL,1,1,1,0,1,1,Month-to-month,1,Credit card (automatic),58.6,1502.65,1
+Male,0,0,0,67,1,0,DSL,0,0,0,1,0,0,Two year,0,Bank transfer (automatic),50.55,3260.1,0
+Male,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,1,Month-to-month,1,Electronic check,35.45,35.45,1
+Male,0,0,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Electronic check,44.35,81.25,1
+Female,0,0,0,43,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),25.7,1188.2,0
+Female,0,0,0,23,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),75,1778.5,0
+Female,0,1,1,64,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,20.2,1277.75,0
+Male,0,0,1,57,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.6,1170.55,0
+Female,1,1,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Mailed check,70.45,70.45,1
+Female,1,1,1,72,1,1,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),88.05,6425.65,0
+Female,0,0,0,8,1,1,DSL,1,0,0,1,0,1,Month-to-month,0,Electronic check,71.15,563.65,1
+Female,0,1,0,61,1,1,Fiber optic,0,1,0,0,1,1,Two year,1,Bank transfer (automatic),101.05,5971.25,0
+Male,0,0,0,64,1,1,Fiber optic,1,1,0,0,0,0,One year,1,Credit card (automatic),84.3,5289.05,0
+Male,1,1,0,71,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),23.95,1756.2,0
+Female,0,1,1,65,1,1,Fiber optic,1,1,0,1,1,0,Two year,1,Credit card (automatic),99.05,6416.7,0
+Male,0,0,0,3,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,19.6,61.35,0
+Male,0,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,45.65,45.65,1
+Male,0,0,1,30,1,0,DSL,0,1,1,0,1,0,One year,0,Credit card (automatic),64.5,1929.95,0
+Male,0,1,1,15,1,0,DSL,0,1,1,1,0,1,Month-to-month,1,Mailed check,69.5,1071.4,0
+Female,0,1,1,8,1,0,DSL,0,0,0,1,1,1,Month-to-month,1,Electronic check,68.55,564.35,0
+Male,0,0,0,7,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,0,Mailed check,95,655.5,1
+Female,0,1,1,70,1,1,Fiber optic,1,1,1,0,1,1,One year,1,Credit card (automatic),108.15,7930.55,0
+Male,0,1,1,62,1,0,DSL,1,1,1,1,1,1,Two year,0,Electronic check,86.1,5215.25,0
+Female,0,1,1,6,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.7,113.5,0
+Female,0,1,1,14,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Bank transfer (automatic),80.9,1152.8,0
+Female,0,0,0,22,1,1,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Bank transfer (automatic),84.15,1821.95,0
+Male,0,1,1,22,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.15,419.9,0
+Male,0,1,1,16,1,0,DSL,1,0,1,0,1,0,Two year,0,Mailed check,64.25,1024,0
+Male,0,0,0,10,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),25.7,251.6,0
+Female,0,0,1,13,1,0,DSL,1,1,0,0,0,0,Month-to-month,1,Electronic check,56,764.55,0
+Female,0,1,0,20,1,1,Fiber optic,1,0,0,0,0,0,Month-to-month,0,Electronic check,82.4,1592.35,1
+Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,69.7,135.2,0
+Male,0,0,0,53,1,1,DSL,0,1,1,1,0,1,Two year,1,Bank transfer (automatic),73.9,3958.25,0
+Female,0,1,1,11,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.6,233.9,0
+Male,0,1,0,69,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.9,1363.45,0
+Female,0,0,0,4,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),70.9,273,1
+Male,0,1,0,72,1,1,DSL,1,1,1,1,1,1,Two year,1,Bank transfer (automatic),89.05,6254.45,0
+Male,1,1,0,58,0,0 phone service,DSL,0,1,1,0,0,1,Month-to-month,1,Electronic check,45.3,2651.2,1
+Female,0,1,1,16,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,20.4,321.4,0
+Male,0,1,0,43,1,0,Fiber optic,1,0,1,1,0,0,One year,1,Bank transfer (automatic),84.25,3539.25,0
+Female,0,1,0,2,1,1,Fiber optic,0,0,1,1,1,1,Month-to-month,1,Electronic check,104.4,242.8,1
+Male,0,1,0,14,1,0,DSL,0,1,1,1,1,1,Two year,0,Mailed check,81.95,1181.75,0
+Female,0,1,0,53,1,0,Fiber optic,0,0,1,0,1,1,One year,1,Bank transfer (automatic),94.85,5000.2,1
+Female,0,0,0,32,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),20.55,654.55,0
+Female,0,1,0,34,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,24.7,780.2,0
+Female,1,0,0,15,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Mailed check,74.45,1145.7,1
+Female,1,0,0,7,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),76.45,503.6,1
+Female,0,1,1,15,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Bank transfer (automatic),105.35,1559.25,0
+Male,0,1,0,61,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Credit card (automatic),20.55,1252,0
+Female,0,0,0,1,0,0 phone service,DSL,0,1,0,0,0,0,Month-to-month,0,Bank transfer (automatic),29.95,29.95,1
+Female,0,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Mailed check,45.3,45.3,0
+Male,0,0,0,8,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Bank transfer (automatic),84.5,662.65,1
+Male,0,1,1,33,1,0,DSL,1,0,1,0,1,1,One year,0,Credit card (automatic),74.75,2453.3,0
+Female,0,0,0,13,1,0,Fiber optic,0,1,1,0,0,0,Month-to-month,1,Electronic check,79.25,1111.65,1
+Female,0,1,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,24.8,24.8,1
+Male,0,0,0,20,1,0,DSL,0,1,0,0,0,0,Month-to-month,1,Mailed check,51.8,1023.85,0
+Male,0,0,0,3,0,0 phone service,DSL,0,0,0,1,0,0,Month-to-month,0,Electronic check,30.4,82.15,0
+Female,0,0,0,13,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Electronic check,19.65,244.8,0
+Female,0,1,0,40,1,1,DSL,0,1,0,0,0,0,Month-to-month,0,Credit card (automatic),56.6,2379.1,0
+Male,0,1,1,43,1,1,DSL,1,0,0,1,0,1,One year,1,Credit card (automatic),71.9,3173.35,0
+Male,0,1,0,6,1,0,Fiber optic,0,1,1,0,0,1,Month-to-month,1,Electronic check,91,531,1
+Female,0,1,1,69,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),19.75,1375.4,0
+Female,0,1,1,72,1,1,Fiber optic,1,1,1,0,1,1,Two year,0,Credit card (automatic),109.7,8129.3,0
+Male,0,1,1,59,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),19.3,1192.7,0
+Female,0,1,0,20,1,0,Fiber optic,1,1,0,1,1,0,Month-to-month,1,Electronic check,96.55,1901.65,0
+Male,0,1,1,24,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),24.1,587.4,0
+Male,0,0,0,59,1,1,Fiber optic,0,1,1,1,1,1,Month-to-month,1,Electronic check,111.35,6519.75,0
+Male,0,1,1,72,1,1,Fiber optic,0,1,1,1,1,1,Two year,1,Bank transfer (automatic),112.25,8041.65,0
+Male,0,0,1,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.75,20.75,0
+Female,0,1,0,27,1,0,Fiber optic,1,0,0,1,1,1,One year,1,Credit card (automatic),101.9,2681.15,0
+Female,0,0,0,14,1,0,Fiber optic,0,1,0,1,0,0,Month-to-month,1,Credit card (automatic),80.05,1112.3,0
+Male,0,1,1,71,1,1,Fiber optic,1,0,1,0,1,1,Two year,0,Electronic check,105.55,7405.5,0
+Male,0,0,1,13,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Credit card (automatic),78.3,1033.95,0
+Male,0,0,0,44,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Credit card (automatic),68.85,2958.95,0
+Female,0,0,0,33,1,0,Fiber optic,0,1,0,1,0,0,One year,1,Bank transfer (automatic),79.95,2684.85,0
+Male,0,1,1,72,0,0 phone service,DSL,1,1,1,1,0,1,Two year,1,Credit card (automatic),55.45,4179.2,0
+Male,0,0,0,1,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,79.9,79.9,1
+Female,0,0,0,19,1,0,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Electronic check,106.6,1934.45,1
+Male,0,1,0,64,1,0,Fiber optic,0,1,1,1,1,1,One year,1,Credit card (automatic),102.45,6654.1,0
+Male,0,0,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Credit card (automatic),46,84.5,1
+Female,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Electronic check,25.25,25.25,0
+Male,0,0,1,61,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.75,1124.2,0
+Female,0,1,1,29,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Electronic check,20,540.05,0
+Male,1,1,0,23,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,86.8,1975.85,0
+Female,0,1,0,57,0,0 phone service,DSL,1,1,1,0,1,1,Month-to-month,1,Bank transfer (automatic),58.75,3437.45,0
+Male,0,1,1,72,0,0 phone service,DSL,1,1,1,1,0,0,Two year,0,Credit card (automatic),45.25,3139.8,0
+Male,0,1,0,66,0,0 phone service,DSL,1,1,1,1,1,0,Two year,1,Bank transfer (automatic),56.6,3789.2,0
+Male,0,1,1,65,1,1,DSL,0,1,1,1,1,1,Two year,1,Bank transfer (automatic),84.2,5324.5,0
+Female,0,0,0,8,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Bank transfer (automatic),80,624.6,0
+Female,0,0,0,4,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,70.15,268.35,1
+Female,0,1,0,71,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),24.75,1836.9,0
+Male,0,1,1,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Electronic check,20.2,20.2,0
+Male,0,1,0,4,1,0,DSL,0,0,0,1,0,0,Month-to-month,0,Mailed check,50.05,179.35,1
+Female,0,0,0,12,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Bank transfer (automatic),19.35,219.35,0
+Male,0,0,0,24,0,0 phone service,DSL,1,0,0,0,1,1,One year,1,Mailed check,50.6,1288.75,0
+Female,0,1,1,31,1,0,DSL,1,0,1,1,1,1,One year,1,Mailed check,81.15,2545.75,0
+Female,0,1,0,1,1,0,DSL,0,0,0,0,1,0,Month-to-month,0,Mailed check,55.2,55.2,1
+Male,0,0,0,30,1,1,Fiber optic,0,0,0,1,0,1,Month-to-month,1,Credit card (automatic),89.9,2723.15,0
+Female,0,1,1,47,1,1,DSL,1,1,0,1,1,1,Two year,1,Credit card (automatic),85.3,4107.25,0
+Male,0,0,0,54,1,1,Fiber optic,1,1,0,1,1,1,Two year,1,Credit card (automatic),108,5760.65,0
+Male,0,0,0,50,1,0,Fiber optic,1,1,0,1,0,1,Month-to-month,1,Credit card (automatic),93.5,4747.5,0
+Male,0,0,0,1,1,0,Fiber optic,1,0,0,0,0,1,Month-to-month,1,Mailed check,84.6,84.6,1
+Female,0,0,0,72,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),20.25,1566.9,0
+Female,0,0,0,29,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,25.15,702,0
+Male,0,0,0,2,1,1,DSL,0,1,0,0,0,0,Month-to-month,1,Mailed check,54.4,114.1,1
+Female,0,0,0,10,0,0 phone service,DSL,0,0,0,1,0,0,Two year,1,Mailed check,29.6,299.05,0
+Male,0,1,0,18,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,73.15,1305.95,0
+Female,1,0,0,11,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Electronic check,95,1120.3,1
+Male,0,0,0,16,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.75,284.35,0
+Female,0,0,0,72,1,1,DSL,0,1,1,1,1,1,Two year,0,Bank transfer (automatic),86.6,6350.5,0
+Male,0,0,0,72,1,1,Fiber optic,0,1,1,1,1,1,Two year,0,Bank transfer (automatic),109.2,7878.3,0
+Female,0,1,1,41,1,0,DSL,1,0,0,1,1,1,One year,1,Credit card (automatic),74.7,3187.65,0
+Female,1,1,0,65,1,0,Fiber optic,1,1,1,0,0,1,Month-to-month,0,Electronic check,94.4,6126.15,0
+Female,1,0,0,13,1,0,DSL,0,1,1,0,0,0,Month-to-month,1,Electronic check,54.8,731.3,0
+Male,1,0,0,4,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,0,Electronic check,75.35,273.4,0
+Male,0,0,0,41,1,1,DSL,0,0,1,0,0,1,One year,0,Mailed check,65,2531.8,0
+Male,1,0,0,15,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Electronic check,74.4,1074.3,1
+Male,0,0,0,1,1,0,DSL,1,0,0,0,0,0,Month-to-month,1,Mailed check,48.55,48.55,1
+Male,0,0,0,42,1,1,Fiber optic,1,1,1,0,0,1,One year,1,Electronic check,99,4298.45,0
+Female,0,1,0,51,1,0,Fiber optic,1,1,1,0,1,0,One year,1,Electronic check,93.5,4619.55,0
+Female,0,1,1,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Electronic check,70.4,147.15,1
+Male,1,1,0,1,0,0 phone service,DSL,0,1,0,0,1,0,Month-to-month,0,Electronic check,40.2,40.2,1
+Male,0,0,1,32,1,1,DSL,0,1,1,1,1,1,One year,0,Credit card (automatic),83.7,2633.3,0
+Male,0,0,0,10,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.85,193.05,0
+Female,0,1,1,67,0,0 phone service,DSL,0,1,1,1,1,1,Two year,1,Electronic check,59.55,4103.9,0
+Female,0,1,1,61,1,1,Fiber optic,1,1,1,1,1,1,Two year,0,Credit card (automatic),115.1,7008.15,0
+Male,0,0,0,50,1,1,Fiber optic,1,1,1,1,1,1,One year,0,Credit card (automatic),114.35,5791.1,0
+Female,0,1,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Credit card (automatic),44.6,80.55,1
+Female,1,1,0,29,0,0 phone service,DSL,1,0,1,0,0,1,Month-to-month,1,Electronic check,45,1228.65,0
+Male,1,0,0,3,0,0 phone service,DSL,0,0,1,0,0,1,Month-to-month,1,Electronic check,41.15,132.2,1
+Male,1,0,0,13,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,106.9,1364.3,1
+Male,1,1,0,57,1,1,Fiber optic,0,1,0,0,0,1,Month-to-month,1,Electronic check,89.85,4925.35,0
+Female,0,0,0,31,0,0 phone service,DSL,1,1,0,1,0,1,Two year,1,Bank transfer (automatic),49.85,1520.1,0
+Female,0,1,0,45,1,1,Fiber optic,1,1,1,1,1,1,One year,0,Mailed check,113.3,5032.25,0
+Female,0,1,1,61,1,1,DSL,1,1,1,1,1,1,Two year,0,Mailed check,88.1,5526.75,0
+Male,0,0,0,50,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),24.9,1195.25,0
+Female,1,0,0,19,1,0,Fiber optic,1,1,0,1,1,1,Month-to-month,1,Mailed check,105,2007.25,0
+Male,0,0,0,59,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.35,1099.6,1
+Female,0,1,0,71,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),24.25,1732.95,0
+Female,1,0,0,16,1,0,Fiber optic,0,0,0,1,1,1,Month-to-month,1,Electronic check,94.45,1511.2,1
+Male,0,1,0,57,1,0,DSL,1,1,0,1,0,0,Two year,1,Electronic check,59.75,3450.15,0
+Male,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),24.8,24.8,1
+Male,1,1,0,20,1,1,Fiber optic,0,1,0,1,1,1,Month-to-month,0,Electronic check,107.05,2172.05,0
+Male,0,0,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Mailed check,70.6,70.6,1
+Male,0,1,0,5,1,1,Fiber optic,0,1,0,1,0,0,Month-to-month,1,Electronic check,85.4,401.1,1
+Female,0,1,0,52,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,105.05,5624.85,1
+Male,1,1,0,21,1,0,DSL,1,1,0,0,0,1,Month-to-month,1,Bank transfer (automatic),64.95,1339.8,0
+Male,0,0,0,14,0,0 phone service,DSL,0,0,1,1,1,1,Month-to-month,1,Electronic check,55,771.95,0
+Female,0,0,0,5,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Bank transfer (automatic),50.55,244.75,0
+Female,0,0,0,6,1,0,DSL,1,1,0,0,0,0,Month-to-month,0,Mailed check,55.15,322.9,0
+Female,0,0,0,10,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Electronic check,51.2,498.25,0
+Female,0,0,1,1,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,25.4,25.4,0
+Female,0,0,0,68,1,1,DSL,1,0,0,0,0,0,Month-to-month,0,Mailed check,54.45,3687.75,0
+Male,0,1,1,18,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,0,Mailed check,95.15,1779.95,1
+Female,0,0,0,22,1,1,Fiber optic,0,0,0,0,0,0,One year,1,Credit card (automatic),76,1783.6,0
+Female,0,0,0,20,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,44.35,927.15,0
+Male,0,1,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Electronic check,70,70,1
+Male,1,0,0,8,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Electronic check,74.5,606.55,1
+Male,0,0,0,10,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Credit card (automatic),44.85,435.4,1
+Male,1,0,0,24,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,76.1,1712.7,1
+Male,0,0,0,35,1,0,DSL,1,1,1,0,0,0,One year,0,Mailed check,61.2,2021.2,0
+Male,0,1,1,23,1,0,Fiber optic,0,1,1,1,0,0,One year,0,Mailed check,86.8,1940.8,0
+Female,1,0,0,6,1,0,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,89.35,567.8,0
+Female,0,0,0,12,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.7,220.35,0
+Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.25,20.25,0
+Female,0,1,1,71,1,0,Fiber optic,0,0,1,0,0,0,Two year,0,Electronic check,76.05,5436.45,0
+Male,1,1,0,35,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,100.8,3437.5,0
+Male,0,1,1,40,1,1,DSL,0,0,1,0,1,1,Month-to-month,1,Electronic check,74.55,3015.75,0
+Female,0,0,1,1,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,1,Mailed check,73.6,73.6,1
+Male,0,0,0,23,1,0,DSL,0,0,0,0,1,1,One year,1,Bank transfer (automatic),64.9,1509.8,0
+Female,1,0,0,4,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,95.45,396.1,1
+Male,0,0,0,4,1,0,Fiber optic,0,0,1,1,0,1,Month-to-month,1,Electronic check,90.4,356.65,0
+Female,0,1,1,68,0,0 phone service,DSL,0,1,1,1,1,1,Two year,0,Credit card (automatic),60.3,4109,0
+Male,0,0,1,38,1,1,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Mailed check,81.85,3141.7,0
+Male,0,1,1,52,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,24.8,1229.1,0
+Male,1,1,0,32,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,74.9,2303.35,1
+Male,0,1,1,29,1,0,DSL,0,0,1,1,1,1,Two year,0,Mailed check,75.55,2054.4,0
+Male,1,1,1,38,1,1,Fiber optic,0,1,0,0,1,1,Month-to-month,0,Electronic check,101.15,3741.85,0
+Male,0,0,1,48,1,1,DSL,1,0,1,0,1,1,One year,1,Credit card (automatic),78.75,3682.45,0
+Female,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Bank transfer (automatic),19.25,19.25,0
+Female,0,0,0,22,1,0,Fiber optic,1,0,0,1,0,1,Month-to-month,1,Electronic check,89.05,1886.25,0
+Female,0,0,0,43,1,1,Fiber optic,1,1,1,1,1,1,Two year,0,Credit card (automatic),115.05,4895.1,0
+Female,1,1,0,5,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,69.35,341.6,0
+Male,0,0,0,5,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,80.6,415.55,1
+Male,0,1,1,51,1,1,Fiber optic,1,1,0,1,1,1,One year,0,Credit card (automatic),110.05,5686.4,0
+Male,0,0,0,71,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),19.9,1355.1,0
+Female,0,1,1,38,1,0,DSL,1,1,0,1,1,1,One year,0,Credit card (automatic),80.3,3058.65,1
+Male,1,1,0,24,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,93.15,2231.05,1
+Male,0,1,0,35,1,1,Fiber optic,0,0,0,1,1,0,Month-to-month,1,Credit card (automatic),91.5,3236.35,0
+Female,1,0,0,54,1,1,DSL,1,1,0,1,1,1,One year,0,Bank transfer (automatic),82.45,4350.1,1
+Female,0,1,1,72,0,0 phone service,DSL,0,1,1,1,1,1,Two year,1,Electronic check,60,4264,0
+Male,0,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,44.8,44.8,0
+Female,0,1,1,9,1,0,DSL,1,0,0,0,0,0,One year,0,Mailed check,48.6,422.3,0
+Male,0,0,0,69,0,0 phone service,DSL,1,1,0,1,1,1,Two year,1,Bank transfer (automatic),60.05,4176.7,0
+Female,0,1,0,52,1,1,Fiber optic,0,1,0,1,1,1,Month-to-month,1,Credit card (automatic),102.7,5138.1,0
+Female,1,1,0,11,1,0,Fiber optic,0,0,0,1,1,0,Month-to-month,1,Electronic check,82.9,880.05,0
+Female,1,0,0,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,70.35,139.05,1
+Female,1,1,1,28,0,0 phone service,DSL,0,0,0,0,1,0,Month-to-month,1,Electronic check,35.9,973.65,0
+Female,1,0,0,17,1,1,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Credit card (automatic),82.65,1470.05,0
+Female,0,1,1,35,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.85,739.35,0
+Female,0,0,0,8,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Mailed check,19.2,161.95,0
+Male,0,0,0,46,1,1,Fiber optic,0,1,1,0,1,0,One year,1,Credit card (automatic),94.9,4422.95,0
+Female,0,0,0,7,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,73.85,511.25,1
+Female,0,0,0,2,1,1,Fiber optic,0,0,1,0,0,0,Month-to-month,0,Mailed check,80.6,155.8,1
+Male,0,1,1,68,1,1,DSL,0,1,1,1,0,1,One year,1,Bank transfer (automatic),75.8,5293.95,1
+Female,0,0,0,43,1,1,Fiber optic,0,1,1,0,1,1,Two year,1,Electronic check,104.6,4759.85,1
+Female,0,0,0,68,1,1,DSL,1,1,1,1,1,1,Two year,1,Electronic check,88.15,6148.45,0
+Female,0,0,0,36,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Credit card (automatic),94.8,3565.65,0
+Male,0,1,0,63,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,103.4,6603,1
+Female,1,0,0,32,1,0,DSL,0,1,0,1,0,0,Month-to-month,0,Bank transfer (automatic),54.65,1830.1,0
+Female,0,1,0,71,1,1,DSL,1,1,1,1,1,1,Two year,0,Credit card (automatic),85.75,6223.8,0
+Female,0,1,1,66,1,1,DSL,1,1,1,0,0,0,Month-to-month,1,Bank transfer (automatic),67.45,4508.65,0
+Male,0,0,0,63,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.5,1328.15,0
+Female,0,0,1,41,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Credit card (automatic),20.25,865,0
+Female,1,0,0,1,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,72.1,72.1,0
+Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,90.4,168.2,1
+Female,0,1,0,70,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),19.45,1303.5,0
+Female,0,0,0,23,0,0 phone service,DSL,0,0,0,0,1,1,Month-to-month,1,Electronic check,44.95,996.85,0
+Female,1,1,0,64,1,0,Fiber optic,1,1,1,0,0,1,One year,0,Bank transfer (automatic),97,6430.9,0
+Male,0,1,1,37,1,1,DSL,1,0,1,1,0,0,Two year,1,Credit card (automatic),62.8,2278.75,0
+Male,0,0,1,17,1,0,DSL,0,0,0,0,0,0,One year,0,Mailed check,44.6,681.4,0
+Female,1,0,0,7,1,0,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,89.15,574.35,0
+Male,0,1,1,4,1,1,Fiber optic,0,1,1,0,0,0,Month-to-month,1,Bank transfer (automatic),84.8,371.9,1
+Female,1,0,0,21,0,0 phone service,DSL,1,0,0,0,1,0,Month-to-month,1,Electronic check,41.9,840.1,1
+Female,0,0,0,10,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Bank transfer (automatic),80.25,846,1
+Female,1,0,0,16,1,1,DSL,1,0,0,0,0,0,Month-to-month,1,Electronic check,54.1,889,0
+Male,0,1,0,64,1,1,Fiber optic,0,1,0,1,1,1,Two year,1,Electronic check,105.25,6823.4,0
+Male,1,1,0,27,0,0 phone service,DSL,1,0,0,0,0,0,Month-to-month,0,Credit card (automatic),30.75,805.1,1
+Male,0,1,1,42,1,1,Fiber optic,1,0,0,0,1,1,One year,0,Electronic check,97.1,4016.75,0
+Male,0,0,1,5,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.2,83.75,0
+Female,0,0,0,41,1,0,Fiber optic,0,1,0,1,1,1,Two year,1,Bank transfer (automatic),98.8,3959.15,0
+Male,0,1,0,58,1,0,DSL,0,0,1,0,0,0,One year,0,Credit card (automatic),50.3,2878.55,0
+Female,0,0,0,47,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,20.55,945.7,0
+Male,0,0,0,18,1,0,Fiber optic,0,0,0,1,0,0,Month-to-month,1,Bank transfer (automatic),75.9,1373.05,0
+Male,0,0,0,5,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,0,Bank transfer (automatic),96.5,492.55,1
+Male,0,1,0,23,1,0,DSL,1,0,0,0,0,1,Month-to-month,1,Bank transfer (automatic),59.95,1406,0
+Female,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.15,19.15,0
+Male,0,1,0,71,1,1,Fiber optic,0,0,1,0,1,1,One year,1,Credit card (automatic),98.65,6962.85,0
+Male,0,1,1,72,1,1,Fiber optic,1,1,1,1,1,1,Two year,1,Credit card (automatic),112.6,8126.65,0
+Male,0,1,1,33,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.6,690.25,0
+Male,0,0,0,2,1,1,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,85.65,181.5,1
+Male,0,0,1,24,0,0 phone service,DSL,1,0,0,1,0,0,Two year,0,Mailed check,35.75,830.8,0
+Female,0,1,1,56,1,1,Fiber optic,0,1,1,1,0,1,One year,1,Credit card (automatic),99.75,5608.4,0
+Male,0,0,0,37,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Credit card (automatic),96.1,3646.8,0
+Female,0,0,0,43,1,1,DSL,0,1,1,1,1,1,One year,0,Credit card (automatic),85.1,3662.25,0
+Male,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Mailed check,25.35,25.35,0
+Female,0,1,0,25,1,1,Fiber optic,0,0,1,1,1,1,Month-to-month,1,Bank transfer (automatic),104.95,2566.5,1
+Female,0,0,0,61,1,1,DSL,1,1,1,1,1,1,Two year,0,Electronic check,89.65,5308.7,0
+Male,0,0,0,17,1,1,Fiber optic,0,0,1,1,0,0,One year,0,Electronic check,86.75,1410.25,0
+Female,0,1,0,41,1,0,Fiber optic,0,1,1,1,0,0,One year,1,Bank transfer (automatic),86.2,3339.05,0
+Male,0,0,0,1,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Mailed check,50.65,50.65,1
+Male,1,1,1,72,0,0 phone service,DSL,1,1,1,1,1,1,Two year,1,Bank transfer (automatic),64.8,4732.35,0
+Male,0,0,0,1,1,1,Fiber optic,0,1,0,0,0,1,Month-to-month,1,Credit card (automatic),90.85,90.85,1
+Male,0,0,0,48,1,1,Fiber optic,0,1,1,0,1,1,Two year,1,Bank transfer (automatic),108.1,5067.45,0
+Male,1,1,0,11,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.95,214.75,1
+Female,1,1,0,55,1,1,Fiber optic,0,1,1,0,0,0,Month-to-month,1,Electronic check,85.45,4874.7,1
+Female,0,0,0,42,0,0 phone service,DSL,1,1,1,1,0,1,One year,1,Electronic check,54.75,2348.45,0
+Male,0,0,0,44,1,1,Fiber optic,0,0,1,0,0,1,Month-to-month,1,Mailed check,90.4,4063,0
+Male,0,0,1,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Electronic check,44,44,0
+Female,0,0,0,27,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Bank transfer (automatic),95.6,2595.25,0
+Male,1,1,0,27,1,1,DSL,0,1,1,1,1,1,Month-to-month,1,Bank transfer (automatic),84.8,2309.55,0
+Female,1,0,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,44.3,89.3,0
+Female,0,0,0,19,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.9,367.55,0
+Female,1,1,0,42,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,95.05,3944.5,0
+Female,0,0,0,66,1,1,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),90.05,5965.95,0
+Male,1,1,0,33,1,1,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Bank transfer (automatic),109.9,3694.7,0
+Female,0,0,0,34,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Credit card (automatic),73.95,2524.45,1
+Male,1,0,0,33,0,0 phone service,DSL,0,0,1,1,1,1,Month-to-month,1,Electronic check,54.6,1803.7,0
+Female,0,1,1,23,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.05,415.1,0
+Female,0,0,0,32,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),19.75,624.15,0
+Female,0,0,0,11,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Credit card (automatic),20.05,237.7,0
+Male,0,1,1,69,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Credit card (automatic),99.45,7007.6,1
+Female,0,1,0,68,1,0,DSL,1,0,0,1,0,0,One year,0,Bank transfer (automatic),55.9,3848.8,0
+Male,0,0,0,20,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.7,419.4,0
+Male,0,0,0,72,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),19.8,1468.75,0
+Male,0,1,1,60,1,1,Fiber optic,1,0,1,0,1,0,One year,1,Bank transfer (automatic),95.4,5812,0
+Male,1,1,1,32,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Mailed check,93.95,2861.45,0
+Female,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.9,19.9,1
+Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.6,19.6,1
+Male,1,0,0,3,1,1,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Electronic check,81.35,233.7,1
+Female,0,1,0,46,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Bank transfer (automatic),24.45,1066.15,0
+Female,0,1,0,29,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,74.95,2149.05,0
+Male,0,0,0,51,1,1,Fiber optic,0,1,1,1,0,0,Month-to-month,1,Electronic check,87.35,4473,0
+Female,1,1,0,48,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),70.65,3545.05,0
+Female,0,1,1,16,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,0,Bank transfer (automatic),73.25,1195.75,0
+Male,0,1,0,70,1,1,Fiber optic,0,1,1,1,0,1,Two year,1,Bank transfer (automatic),98.7,6858.9,0
+Female,0,1,1,40,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,24.8,1024.7,0
+Female,0,1,1,22,1,0,Fiber optic,0,1,0,0,0,1,Month-to-month,0,Electronic check,83.3,1845.9,1
+Female,1,0,0,1,1,0,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Bank transfer (automatic),75.3,75.3,1
+Female,0,1,1,5,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,24.3,132.25,0
+Female,0,0,0,7,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),69.85,515.45,0
+Male,0,0,1,29,1,1,Fiber optic,0,1,1,1,0,1,One year,0,Credit card (automatic),100.55,2830.45,0
+Male,0,1,1,44,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,25.7,1110.5,0
+Female,0,0,0,10,0,0 phone service,DSL,1,1,1,0,0,0,Month-to-month,0,Bank transfer (automatic),40.7,449.3,0
+Female,1,1,0,55,1,1,DSL,0,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),51.65,2838.55,0
+Male,1,1,0,52,1,1,Fiber optic,0,1,1,0,1,1,One year,1,Bank transfer (automatic),105.1,5376.4,0
+Female,0,1,1,10,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,85.95,858.6,0
+Male,0,0,0,18,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),75.6,1395.05,0
+Female,0,1,0,68,1,1,DSL,1,0,0,1,0,0,One year,0,Bank transfer (automatic),58.25,3975.7,0
+Female,0,1,1,61,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.4,1182.55,1
+Female,0,1,1,72,0,0 phone service,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),65.2,4784.45,0
+Male,0,0,0,2,1,0,DSL,1,0,1,0,0,0,Month-to-month,0,Electronic check,53.45,119.5,0
+Male,0,0,0,12,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),45.4,518.9,1
+Male,0,1,0,41,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Electronic check,19.75,899.45,0
+Female,0,0,0,26,0,0 phone service,DSL,0,0,1,1,1,0,Month-to-month,1,Credit card (automatic),44.45,1183.8,0
+Male,0,1,0,36,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),20.85,720.05,0
+Male,0,1,1,72,1,1,Fiber optic,1,1,1,1,1,1,Two year,1,Credit card (automatic),114.05,8468.2,0
+Male,0,1,0,35,1,1,Fiber optic,0,0,0,1,0,1,Month-to-month,1,Bank transfer (automatic),89.85,3161.2,0
+Male,0,0,0,1,1,0,DSL,0,0,1,1,0,0,Month-to-month,0,Mailed check,55.05,55.05,0
+Female,0,1,1,16,1,1,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Electronic check,112.95,1882.55,0
+Female,1,0,0,49,1,1,Fiber optic,1,0,0,0,1,1,Month-to-month,0,Credit card (automatic),101.55,5070.4,0
+Female,0,1,0,54,1,1,Fiber optic,1,1,1,1,1,1,One year,1,Mailed check,114.65,6049.5,0
+Female,0,0,1,18,1,1,DSL,1,0,0,0,0,1,Month-to-month,0,Electronic check,64.8,1166.7,0
+Female,0,1,0,36,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,80.4,2937.65,0
+Female,0,0,0,60,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,105.9,6396.45,1
+Female,0,0,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Mailed check,69.55,69.55,1
+Female,0,1,1,52,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),25.05,1270.25,0
+Male,0,1,1,8,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,94.75,759.55,0
+Male,0,1,1,72,1,0,Fiber optic,1,1,0,1,1,1,Two year,0,Bank transfer (automatic),105.5,7611.55,0
+Female,0,0,0,64,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,24.7,1642.75,0
+\.
+
+
+select xgboost_sk_classifier('telecom', 'telecom_mode', '"customerID"', '"Churn"', '"MonthlyCharges", "TotalCharges"', NULL, $${'booster': ['gbtree'], 'eta': (0.1), 'max_depth': 5, 'objective': ['binary:logistic']}$$, TRUE);
+\d+ telecom_mode
+select xgboost_sk_predict('telecom', 'telecom_mode', 'outtable', '"customerID"', TRUE);
\ No newline at end of file
diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.py_in b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.py_in
new file mode 100644
index 0000000..00cc866
--- /dev/null
+++ b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.py_in
@@ -0,0 +1,488 @@
+# -*- coding:utf-8 -*-
+# Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+#
+# openGauss is licensed under Mulan PSL v2.
+# You can use this software according to the terms
+# and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+# http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------
+# Xgboost regression and Classification use sklearn API.
+# ----------------------------------------------------------------------
+import ast
+import collections
+import itertools
+import plpy
+
+import cPickle as pickle
+import pandas as pd
+
+from utilities.validate_args import is_var_valid
+from utilities.validate_args import table_exists
+from utilities.validate_args import table_is_empty
+from utilities.validate_args import columns_exist_in_table
+from utilities.validate_args import get_cols
+from utilities.validate_args import unquote_ident
+from utilities.validate_args import quote_ident
+from utilities.utilities import _assert
+from utilities.control import MinWarning
+
+from xgboost_utilities import get_features_to_use
+
+
+def xgboost_sk_classifier(schema_madlib, input_table, model_table, id_column, dependent_varname,
+ str_of_features, str_of_features_to_exclude, params_str, evaluation, verbose, **kwargs):
+ """
+ Args:
+ @param input_table, -- name of input table
+ @param model_table, -- name of output table
+ @param id_column
+ @param dependent_varname, -- name of dependent variable Y
+ @param str_of_features, -- name of independent variables X, Comma separated column names to be
+ used as the predictors, can be '*'
+ to include all columns except the dependent_variable
+ @param str_of_features_to_exclude -- Comma separated column names to be excluded if str_of_features is '*',
+ Ignore 'str_of_features_to_exclude' if 'str_of_features' is not '*'
+ @param params_str, -- names of paraters for xgboost
+ @param verbose -- verbose or not
+ """
+ with MinWarning('warning'):
+ _validate_args(schema_madlib, input_table, model_table, id_column,
+ dependent_varname, str_of_features, str_of_features_to_exclude, params_str)
+
+ _, features_list = get_features_to_use(schema_madlib, input_table, id_column, dependent_varname,
+ str_of_features, str_of_features_to_exclude, verbose)
+
+ sql = '''SELECT {schema_madlib}.xgboost_grid_search('{input_table}', '{grid_search_results_tbl}', '{id_column}',
+ '{dependent_varname}', ARRAY{features_list}, '{xgb_type}', $${params_str}$$, {evaluation}, {verbose});
+ '''.format(
+ schema_madlib=schema_madlib,
+ input_table=input_table,
+ grid_search_results_tbl=model_table,
+ id_column=id_column,
+ dependent_varname=dependent_varname,
+ features_list=features_list,
+ xgb_type='C',
+ params_str=params_str,
+ evaluation=evaluation,
+ verbose=verbose
+ )
+ if verbose:
+ plpy.info(sql)
+
+ ret = plpy.execute(sql)
+ return ret[0]['xgboost_grid_search']
+
+
+def xgboost_sk_regressor(schema_madlib, input_table, model_table, id_column,
+ dependent_varname, str_of_features, str_of_features_to_exclude, params_str, evaluation, verbose, **kwargs):
+ """
+ Args:
+ @param input_table, -- name of input table
+ @param model_table, -- name of output table
+ @param id_column -- id column of the table
+ @param dependent_varname, -- name of dependent variable Y
+ @param str_of_features, -- name of independent variables X
+ @param params_str, -- {key: [value1, valule2]}
+ @param verbose
+ """
+ with MinWarning('warning'):
+ _validate_args(schema_madlib, input_table, model_table, id_column,
+ dependent_varname, str_of_features, str_of_features_to_exclude, params_str)
+
+ _, features_list = get_features_to_use(schema_madlib, input_table, id_column, dependent_varname,
+ str_of_features, str_of_features_to_exclude, verbose)
+
+ sql = '''SELECT {schema_madlib}.xgboost_grid_search('{input_table}', '{grid_search_results_tbl}', '{id_column}',
+ '{dependent_varname}', ARRAY{features_list}, '{xgb_type}', $${params_str}$$, {evaluation}, {verbose});
+ '''.format(
+ schema_madlib=schema_madlib,
+ input_table=input_table,
+ grid_search_results_tbl=model_table,
+ id_column=id_column,
+ dependent_varname=dependent_varname,
+ features_list=features_list,
+ xgb_type='R',
+ params_str=params_str,
+ evaluation=evaluation,
+ verbose=verbose
+ )
+ if verbose:
+ plpy.info(sql)
+
+ ret = plpy.execute(sql)
+ return ret[0]['xgboost_grid_search']
+
+
+def xgboost_sk_predict(schema_madlib, input_table, model_table, output_table, id_column, verbose, **kwargs):
+ """
+ Args:
+ @param input_table, -- Input: name of input table
+ @param model_table, -- Input: name of model table
+ @param output_table, -- Output: save the results.
+ @param id_column
+ @param verbose
+ """
+ with MinWarning('warning'):
+ _validate_predict(schema_madlib, input_table, model_table, output_table, id_column)
+
+ sql = '''SELECT y_type, features, model FROM {model_table}'''.format(
+ model_table=model_table)
+ result = plpy.execute(sql)
+
+ # get the type of label.
+ y_type = result[0]['y_type']
+ features_list = result[0]['features']
+ features_list = [quote_ident(x) for x in features_list]
+
+ # 1) Extract feature names from information_schema
+ all_columns = get_cols(input_table)
+
+ for f in features_list:
+ if f not in all_columns:
+ plpy.error('''xgboost_sk_predict, {f} donot find in training table.'''.format(str(f)))
+
+ # 2) get model
+ model = result[0]['model']
+ model = pickle.loads(model)
+
+ # 3) Fetch dataset for model training
+ sql = """select {col} from {input_table} order by {id_column}""".format(
+ id_column=id_column,
+ col=','.join(features_list),
+ input_table=input_table
+ )
+
+ if verbose:
+ plpy.info("xgboost_sk_predict() " + sql)
+
+ result = plpy.execute(sql)
+ test_df = pd.DataFrame.from_records(result)
+
+ features_list = [unquote_ident(x) for x in features_list]
+ features = filter(lambda x: x in test_df.columns, features_list)
+ if verbose:
+ plpy.info("xgboost_sk_predict() - features: " + str(features))
+ test_df = test_df[features]
+
+ # get id
+ sql = """select {id_column} from {input_table} order by {id_column}""".format(
+ id_column=id_column,
+ input_table=input_table
+ )
+ result = plpy.execute(sql)
+
+ id_df = pd.DataFrame.from_records(result)
+ id_df = id_df.reset_index(drop=True)
+ id_df = id_df[id_df.columns[0]]
+
+ # start to predict
+ predict_res = model.predict(test_df)
+
+ sql = '''CREATE TABLE {output_table} (id bigint, prediction {y_type})'''.format(
+ output_table=output_table, y_type=y_type)
+ if verbose:
+ plpy.info("xgboost_sk_predict() " + sql)
+
+ plpy.execute(sql)
+ _assert(len(predict_res) == len(id_df),
+ "Xgboost predict error: len(id) != len(predict)")
+
+ sql = """INSERT INTO {output_table} VALUES ({id}, {prediction});"""
+ for i in range(len(id_df)):
+ plpy.execute(
+ sql.format(id=id_df[i],
+ prediction=predict_res[i],
+ output_table=output_table
+ )
+ )
+
+# ------------------------------------------------------------------------------
+# -- validate args ------------------------------------------------------------
+# ------------------------------------------------------------------------------
+def _validate_args(schema_madlib, input_table, model_table, id_column,
+ dependent_varname, str_of_features, str_of_features_to_exclude, params_str):
+ """
+ @brief validate the arguments
+ """
+ # 1, check input_table: name.
+ _assert(input_table and input_table.strip().lower() not in ('null', ''),
+ "Xgboost error: Invalid data table name!")
+ _assert(table_exists(input_table),
+ "Xgboost error: Data table does not exist!")
+ _assert(not table_is_empty(input_table),
+ "Xgboost error: Data table is empty!")
+
+ # 2, check input table: id, Dependent variable, features.
+ _assert(columns_exist_in_table(input_table, [id_column]),
+ "Xgboost error: id_column doesnot exist in the input_table")
+
+ _assert(dependent_varname is None or dependent_varname.strip().lower() not in ('null', ''),
+ "Xgboost error: Invalid dependent column name!")
+
+ _assert(is_var_valid(input_table, dependent_varname),
+ "Xgboost error: Invalid dependent variable ({0}).".format(dependent_varname))
+
+ _assert(str_of_features and str_of_features.strip(),
+ "Xgboost error: Features to include is empty.")
+
+ if str_of_features.strip() != '*':
+ _assert(is_var_valid(input_table, str_of_features),
+ "Xgboost error: Invalid feature list ({0})".format(str_of_features))
+
+ if str_of_features.strip() != '*':
+ _assert(dependent_varname not in str_of_features,
+ "Xgboost error: Dependent_varname in str_of_features")
+
+ # 3, check output table.
+ _assert(model_table is not None and model_table.strip().lower() not in ('null', ''),
+ "Xgboost error: Invalid output table name!")
+ _assert(not table_exists(model_table, only_first_schema=True),
+ "Output table name already exists. Drop the table before calling the function.")
+
+ # 4, check params_str
+ if params_str != '':
+ # let xgboost to check the parameter
+ pass
+
+
+def _validate_predict(schema_madlib, input_table, model_table, output_table, id_column):
+ # TODO, check all tables.
+ _assert(input_table is not None and input_table.strip().lower()
+ not in ('null', ''), "Xgboost error: Invalid data table name!")
+ _assert(table_exists(input_table),
+ "Xgboost error: Data table does not exist!")
+ _assert(not table_is_empty(input_table),
+ "Xgboost error: Data table is empty!")
+
+ _assert(model_table is not None and model_table.strip().lower()
+ not in ('null', ''), "Xgboost error: Invalid Model table name!")
+ _assert(table_exists(model_table),
+ "Xgboost error: Model table does not exist!")
+ _assert(not table_is_empty(model_table),
+ "Xgboost error: Model table is empty!")
+ _assert(output_table is not None and output_table.strip().lower()
+ not in ('null', ''), "Xgboost error: Invalid output table name!")
+ _assert(not table_exists(output_table, only_first_schema=True),
+ "Output table name already exists. Drop the table before calling the function.")
+
+# ------------------------------------------------------------------------------
+# -- Online help function ------------------------------------------------------
+# ------------------------------------------------------------------------------
+def xgboost_sk_classifier_help_message(schema_madlib, message, **kwargs):
+ """ Help message for Xgboost XGBClassifier sklearn API
+
+ @brief
+ Args:
+ @param schema_madlib string, Name of the schema madlib
+ @param message string, Help message indicator
+
+ Returns:
+ String. Contain the help message string
+ """
+ if not message:
+ help_string="""
+ -----------------------------------------------------------------------
+ SUMMARY
+ -----------------------------------------------------------------------
+ XGBoost sklearn API Training Library.
+
+ For more details on function usage:
+ SELECT {schema_madlib}.xgboost_sk_classifier('usage')
+ """
+ elif message in ['usage', 'help', '?']:
+ help_string="""
+ -----------------------------------------------------------------------
+ USAGE
+ -----------------------------------------------------------------------
+ SELECT {schema_madlib}.xgboost_sk_classifier(
+ input_table TEXT, -- in, Data table name
+ model_table TEXT, -- out, save model and features
+ id_column TEXT, -- id column of the dataset table
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X, Comma separated column names to be
+ used as the predictors, can be '*'
+ to include all columns except the
+ dependent_variable
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to
+ exlude from the predictors list,
+ if str_of_features is '*'
+ params_str TEXT, -- xgboost parameters
+ evaluation BOOL, -- evaluation the model? default FALSE
+ verbose bool -- Boolean, whether to print more info, default is False
+ );
+
+ -----------------------------------------------------------------------
+ parameters
+ -----------------------------------------------------------------------
+ below is optional parameters:
+
+ evalution:
+ default is FALSE, if you set evaluation=TRUE, it will split the training dataset(0.8 for training, 0.2 for testing),
+ and calculate metrics.
+ For binary classification: ['precision', 'recall', 'fscore', 'support']
+ OR for multi-calssification: ['kappa', 'acc']
+
+ params_str:
+ For example, if the input is:
+ $${{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': [5,6,7], 'objective': ['binary:logistic']}}$$
+ OR
+ $${{'booster': ('gbtree',), 'eta': 0.1, 'max_depth': (5,6,7), 'objective': ('binary:logistic',)}}$$
+ Then, cartesian_product is:
+ [('booster = gbtree', 'eta = 0.1', 'max_depth = 5', 'objective = binary:logistic'),
+ ('booster = gbtree', 'eta = 0.1', 'max_depth = 6', 'objective = binary:logistic'),
+ ('booster = gbtree', 'eta = 0.1', 'max_depth = 7', 'objective = binary:logistic')]
+
+ select {schema_madlib}.xgboost_sk_Classifier('inputTable', 'output_modelTable', 'id', 'Y', 'x1, x2, x3', NULL,
+ $${{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': 5, 'objective': ('binary:logistic',)}}$$, '', TRUE, TRUE);
+
+ -----------------------------------------------------------------------
+ OUTPUT
+ -----------------------------------------------------------------------
+ The output table ('model_table' above) has the following columns:
+ id -- index
+ train_timestamp -- training time
+ model_name -- model name
+ y_type -- the type of independent variable
+ metrics -- default is OFF, evaluation, for binary classification: ['precision', 'recall', 'fscore', 'support']
+ OR for multi-calssification: ['kappa', 'acc']
+ features -- feature in binary model
+ model -- binary model
+ params -- xgboost prarms, {{key:value}}
+ """
+ else:
+ help_string="No such option. Use {schema_madlib}.xgboost_sk_classifier()"
+
+ return help_string.format(schema_madlib=schema_madlib)
+
+
+def xgboost_sk_regressor_help_message(schema_madlib, message, **kwargs):
+ """ Help message for Xgboost XGBRegressor sklearn API
+
+ @brief
+ Args:
+ @param schema_madlib string, Name of the schema madlib
+ @param message string, Help message indicator
+
+ Returns:
+ String. Contain the help message string
+ """
+ if not message:
+ help_string="""
+ -----------------------------------------------------------------------
+ SUMMARY
+ -----------------------------------------------------------------------
+ XGBoost sklearn API Training Library.
+
+ For more details on function usage:
+ SELECT {schema_madlib}.xgboost_sk_regressor('usage')
+ """
+ elif message in ['usage', 'help', '?']:
+ help_string="""
+ -----------------------------------------------------------------------
+ USAGE
+ -----------------------------------------------------------------------
+ SELECT {schema_madlib}.xgboost_sk_regressor(
+ input_table TEXT, -- in, dataset table
+ model_table TEXT, -- out, save model and features
+ id_column TEXT, -- id column of the dataset table
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X, Comma separated column names to be
+ used as the predictors, can be '*'
+ to include all columns except the
+ dependent_variable
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to
+ exlude from the predictors list, if str_of_features is '*'
+ params_str TEXT, -- xgboost parameters
+ evaluation BOOL, -- evaluation the model? default FALSE
+ verbose BOOL -- Boolean, whether to print more info, default is False
+ );
+
+ -----------------------------------------------------------------------
+ parameters
+ -----------------------------------------------------------------------
+ below is optional parameters:
+
+ evalution:
+ default is FALSE, if you set evaluation=TRUE, it will split the training dataset(0.8 for training, 0.2 for testing),
+ and calculate metrics: ['mae', 'mse', 'R2squared', 'rmse']
+
+ params_str:
+ For example, if the input is:
+ $${{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': [5,6,7], 'objective':['reg:linear']}}$$
+ OR
+ $${{'booster': ('gbtree',), 'eta': 0.1, 'max_depth': (5,6,7), 'objective':('reg:linear')}}$$
+ Then, cartesian_product is:
+ [('booster = gbtree', 'eta = 0.1', 'max_depth = 5', 'objective = reg:linear'),
+ ('booster = gbtree', 'eta = 0.1', 'max_depth = 6', 'objective = reg:linear'),
+ ('booster = gbtree', 'eta = 0.1', 'max_depth = 7', 'objective = reg:linear')]
+
+ select {schema_madlib}.xgboost_sk_Regressor('input_TABLE', 'output_model_table', 'id', 'Y', 'x1, x2, x3, x4', 'x5, x6',
+ $${{'booster': ['gbtree'], 'eta': (0.1), 'max_depth': (5,6,7), 'objective':['reg:linear']}}$$, TRUE, TRUE);
+
+ -----------------------------------------------------------------------
+ OUTPUT
+ -----------------------------------------------------------------------
+ The output table ('model_table' above) has the following columns:
+ train_timestamp -- training time
+ model_name -- model name
+ y_type -- the type of independent variable
+ metrics -- if evaluation is TRUE, you will get ['mae', 'mse', 'R2squared', 'rmse']
+ features -- feature in binary model
+ model -- binary model
+ params -- xgboost prarms, {{key:value}}
+ params_indx -- index
+ """
+ else:
+ help_string="No such option. Use {schema_madlib}.xgboost_sk_regressor()"
+
+ return help_string.format(schema_madlib=schema_madlib)
+
+
+def xgboost_sk_predict_help_message(schema_madlib, message, **kwargs):
+ """ Help message for Xgboost predict sklearn API
+
+ @brief
+ Args:
+ @param schema_madlib string, Name of the schema madlib
+ @param message string, Help message indicator
+
+ Returns:
+ String. Contain the help message string
+ """
+ if not message:
+ help_string="""
+ -----------------------------------------------------------------------
+ SUMMARY
+ -----------------------------------------------------------------------
+ XGBoost sklearn API Training Library.
+
+ For more details on function usage:
+ SELECT {schema_madlib}.xgboost_sk_predict('usage')
+ """
+ elif message in ['usage', 'help', '?']:
+ help_string="""
+ -----------------------------------------------------------------------
+ USAGE
+ -----------------------------------------------------------------------
+ SELECT {schema_madlib}.xgboost_sk_predict(
+ input_table TEXT, -- in, test dataset
+ model_table TEXT, -- in, model table from training
+ output_table TEXT, -- out, prediction of the test
+ id_column TEXT, -- id column of the test dataset
+ verbose BOOL -- Boolean, whether to print more info, default is False
+ );
+ """
+ else:
+ help_string="No such option. Use {schema_madlib}.xgboost_sk_predict()"
+
+ return help_string.format(schema_madlib=schema_madlib)
diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.sql_in b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.sql_in
new file mode 100644
index 0000000..19df7d4
--- /dev/null
+++ b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.sql_in
@@ -0,0 +1,197 @@
+/*
+ * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+ *
+ * openGauss is licensed under Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *
+ * http://license.coscl.org.cn/MulanPSL2
+ *
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * -------------------------------------------------------------------------
+ */
+
+m4_include(`SQLCommon.m4')
+
+----------------------------------------------------------------------------------------------
+------ Help messages
+----------------------------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier(
+ message TEXT
+) RETURNS TEXT AS $XX$
+ PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_classifier_help_message)
+$XX$ LANGUAGE plpythonu IMMUTABLE;
+-------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier()
+RETURNS TEXT AS $XX$
+ SELECT MADLIB_SCHEMA.xgboost_sk_classifier(''::TEXT);
+$XX$
+LANGUAGE sql IMMUTABLE;
+
+-----------------------------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor(
+ message TEXT
+) RETURNS TEXT AS $XX$
+ PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_regressor_help_message)
+$XX$ LANGUAGE plpythonu IMMUTABLE;
+-------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor()
+RETURNS TEXT
+AS $XX$
+ SELECT MADLIB_SCHEMA.xgboost_sk_regressor(''::TEXT);
+$XX$
+LANGUAGE sql IMMUTABLE;
+----------------------------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict(
+ message TEXT
+) RETURNS TEXT AS $$
+ PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_predict_help_message)
+$$ LANGUAGE plpythonu IMMUTABLE;
+-------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict()
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_predict(''::TEXT);
+$$
+LANGUAGE sql IMMUTABLE;
+-------------------------------------------------------------
+
+
+----------------------------------------------------------------------------------------------
+------ xgboost Classifier Algo
+----------------------------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier(
+ input_table TEXT, -- in, dataset table
+ model_table TEXT, -- out, save model and features
+ id_column TEXT, -- id column of the dataset table
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list.
+ params_str TEXT, -- xgboost parameters
+ evaluation BOOL, -- evaluation the training model, default FALSE.
+ verbose BOOL
+)
+RETURNS TEXT AS $$
+PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_classifier)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier(
+ input_table TEXT,
+ model_table TEXT,
+ id_column TEXT,
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list.
+ params_str TEXT,
+ evaluation BOOL -- evaluation the training model, default FALSE.
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_classifier($1, $2, $3, $4, $5, $6, $7, $8, FALSE);
+$$ LANGUAGE sql;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier(
+ input_table TEXT,
+ model_table TEXT,
+ id_column TEXT,
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list.
+ params_str TEXT
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_classifier($1, $2, $3, $4, $5, $6, $7, FALSE);
+$$ LANGUAGE sql;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier(
+ input_table TEXT,
+ model_table TEXT,
+ id_column TEXT,
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT -- List of column names (comma-separated string) to exlude from the predictors list.
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_classifier($1, $2, $3, $4, $5, $6, '');
+$$ LANGUAGE sql;
+
+----------------------------------------------------------------------------------------------
+------ xgboost XGBRegressor
+----------------------------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor(
+ input_table TEXT, -- in, dataset table
+ model_table TEXT, -- out, save model and features
+ id_column TEXT, -- id column of the dataset table
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list.
+ params_str TEXT, -- xgboost parameters
+ evaluation BOOL, -- evaluation the training model, default FALSE.
+ verbose BOOL
+)
+RETURNS TEXT AS $$
+PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_regressor)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor(
+ input_table TEXT,
+ model_table TEXT,
+ id_column TEXT,
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list.
+ params_str TEXT,
+ evaluation BOOL -- evaluation the training model, default FALSE.
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_regressor($1, $2, $3, $4, $5, $6, $7, $8, FALSE);
+$$ LANGUAGE sql;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor(
+ input_table TEXT,
+ model_table TEXT,
+ id_column TEXT,
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list.
+ params_str TEXT
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_regressor($1, $2, $3, $4, $5, $6, $7, FALSE);
+$$ LANGUAGE sql;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor(
+ input_table TEXT,
+ model_table TEXT,
+ id_column TEXT,
+ dependent_varname TEXT, -- y, class label
+ str_of_features TEXT, -- X,
+ str_of_features_to_exclude TEXT -- List of column names (comma-separated string) to exlude from the predictors list.
+)
+RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_regressor($1, $2, $3, $4, $5, $6, '');
+$$ LANGUAGE sql;
+
+------------------------------------------------
+-- XGBoost predict
+------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict(
+ input_table TEXT, -- in, test dataset
+ model_table TEXT, -- in, model table from training
+ output_table TEXT, -- out, prediction of the test
+ id_column TEXT, -- id column of the test dataset
+ verbose BOOL
+) RETURNS TEXT AS $$
+PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_predict)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict(
+ input_table TEXT,
+ model_table TEXT,
+ output_table TEXT,
+ id_column TEXT
+) RETURNS TEXT AS $$
+ SELECT MADLIB_SCHEMA.xgboost_sk_predict($1, $2, $3, $4, FALSE);
+$$ LANGUAGE sql;
diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.py_in b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.py_in
new file mode 100644
index 0000000..2355ba7
--- /dev/null
+++ b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.py_in
@@ -0,0 +1,420 @@
+# -*- coding:utf-8 -*-
+# Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+#
+# openGauss is licensed under Mulan PSL v2.
+# You can use this software according to the terms
+# and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+# http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
+# WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------
+# Xgboost regression and Classification use sklearn API.
+# ----------------------------------------------------------------------
+import ast
+import collections
+import itertools
+import plpy
+import re
+
+import xgboost as xgb
+import numpy as np
+import cPickle as pickle
+import pandas as pd
+
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import precision_recall_fscore_support
+from sklearn.metrics import mean_squared_error
+from sklearn.metrics import mean_absolute_error
+from sklearn.metrics import r2_score
+from sklearn.metrics import cohen_kappa_score
+from sklearn.metrics import accuracy_score
+
+from utilities.utilities import unique_string
+from utilities.utilities import split_quoted_delimited_str
+from utilities.utilities import _assert
+from utilities.control import MinWarning
+from utilities.validate_args import get_cols
+from utilities.validate_args import get_expr_type
+from utilities.validate_args import quote_ident
+from utilities.validate_args import unquote_ident
+from utilities.validate_args import _get_table_schema_names
+
+# xgboost donot support category, xgboost only deals with numeric columns
+support_type_list = ('TINYINT', 'SMALLINT', 'INTEGER', 'BINARY_INTEGER', 'BIGINT', 'DECIMAL',
+ 'SMALLSERIAL', 'SERIAL', 'BIGSERIAL',
+ 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'FLOAT8', 'FLOAT', 'BINARY_DOUBLE', 'DEC',
+ 'BOOLEAN')
+
+
+def get_features_to_use(schema_madlib, input_table, id_column, dependent_varname, str_of_features, str_of_features_to_exclude, verbose=False):
+ """
+ get all columns and used columns.
+ return: quote_idented column.
+ """
+ other_col_set = set([id_column, dependent_varname])
+ other_col_set |= set(unquote_ident(i) for i in [id_column, dependent_varname])
+
+ if str_of_features.strip() == '*':
+ all_col = get_cols(input_table) # get all columns in order with quote_ident!
+ all_col_set = set(all_col)
+ exclude_set = set(split_quoted_delimited_str(str_of_features_to_exclude))
+ feature_set = all_col_set - exclude_set
+ filtered_feature_list = list(feature_set - other_col_set)
+ filtered_feature_list = [t for t in all_col if t in filtered_feature_list] # columns order is important
+ else:
+ all_col_set = set(get_cols(input_table))
+ feature_list = split_quoted_delimited_str(str_of_features)
+ feature_exclude = split_quoted_delimited_str(str_of_features_to_exclude)
+ return_set = set(feature_list) - set(feature_exclude) - other_col_set
+ # instead of returning list(return_set) we create a list that has
+ # elements in same order as original feature_list
+ filtered_feature_list = [feat for feat in feature_list if feat in return_set]
+
+ if verbose:
+ plpy.info("get_features_to_use() " + str(all_col_set) + " -- " + str(filtered_feature_list))
+ return all_col_set, filtered_feature_list
+
+def get_column_and_ret_type(schema_madlib, input_table, id_column, dependent_varname, features_list, verbose=False):
+ '''
+ 1. check the data type of the input table.
+ 2. get the type of dependent_varname.
+ '''
+ # Extract feature names from information_schema, and check type.
+ col_list = features_list + [dependent_varname] # X + [y] with unquote_idented
+
+ # input_table = <s_name>.<t_name>
+ s_name, t_name = _get_table_schema_names(input_table, True)
+ sql = """SELECT
+ column_name, data_type
+ FROM
+ information_schema.columns
+ WHERE
+ table_name = '{t_name}' AND table_schema IN {s_name} AND column_name IN {col_list};
+ """.format(
+ t_name=t_name,
+ s_name=s_name,
+ col_list=str(col_list).replace('[', '(').replace(']', ')').replace('"', '')
+ )
+ if verbose:
+ plpy.info("get_column_and_ret_type() " + sql)
+
+ result = plpy.execute(sql)
+
+ # xgboost do not support category, xgboost only deals with numeric columns
+ for r in result:
+ if verbose:
+ plpy.info("get_column_and_ret_type() check type: " + r['column_name'] + " --> " + r['data_type'])
+ if r['data_type'].upper() not in support_type_list:
+ plpy.error("Wrong Xgboost type, " + r['column_name'] + " is " + r['data_type'])
+
+ # save the type of dependent_variable.
+ y_type = get_expr_type(dependent_varname, input_table)
+
+ if verbose:
+ plpy.info("get_column_and_ret_type() lable type is: " + str(y_type))
+
+ return y_type
+
+
+def save_table_as_bytea(schema_madlib, input_table, id_column,
+ dependent_varname, features_list, verbose, **kwargs):
+ """
+ transfrom table => dataframe => dumpfile
+ """
+ # 1) Extract feature names from information_schema
+ col_list = features_list + [dependent_varname] # X + [y], quote_idented
+
+ # 2) Fetch dataset for model training
+ sql = """select {id_column}, {col} from {input_table}""".format(
+ id_column=id_column,
+ col=','.join(col_list),
+ input_table=input_table
+ )
+
+ if verbose:
+ plpy.info("save_table_as_bytea() " + str(sql))
+
+ result = plpy.execute(sql)
+ # => df
+ df = pd.DataFrame.from_records(result)
+
+ if verbose:
+ plpy.info("save_table_as_bytea() " + str(df.columns))
+
+ # dump dataframe.
+ return pickle.dumps(df)
+
+
+def xgboost_train_parallel(schema_madlib, dframe, dependent_varname, features_list,
+ xgb_type, params, evaluation, verbose, **kwargs):
+ """
+ xgboost grid search Algorithm
+ Args:
+ @param dframe, -- name of input table
+ @param dependent_varname, -- name of dependent variable Y
+ @param features_list, -- name of independent variables X
+ @param xgb_type, -- 'C' or 'R', classificaiton or Regression
+ @param params, -- dictionary. params = "{'eta':0.1, 'max_depth':2, 'min_child_weight':3}"
+ """
+ # 1) Load the dataset for model training
+ df = pickle.loads(dframe)
+
+ # 2) Train XGBoost model & return a serialized representation to store in table, keep the order.
+ features_list = [unquote_ident(x) for x in features_list]
+ features = filter(lambda x: x in df.columns, features_list)
+ if verbose:
+ plpy.info("xgboost_train_parallel(): " + str(df.columns) + " <>" + str(features))
+ X = df[features]
+ y = df[unquote_ident(dependent_varname)]
+
+ if evaluation:
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
+ else:
+ X_train, y_train = X, y
+
+ if verbose:
+ plpy.info("xgboost_train_parallel() X_train.columns: " + str(X_train.columns))
+ plpy.info("xgboost_train_parallel() X_train.dtypes: " + str(X_train.dtypes))
+
+ _assert(xgb_type in ('c', 'C', 'R', 'r'),
+ "Xgboost error: xgb_type must be 'C' or 'R'")
+
+ # Train gradient boosted trees
+ # tuple string => dictionary
+ if type(params) == str:
+ params = ast.literal_eval(params)
+ if type(params) != tuple:
+ plpy.error("Wrong XGboost parameters " + str(params))
+
+ params_dict = dict()
+ for p in params:
+ k, v = p.split("=")
+ params_dict[k.strip()] = v.strip()
+
+ if verbose:
+ plpy.info("xgboost_train_parallel() params list: " + str(params_dict))
+ plpy.info("xgboost_train_parallel() train type: " + xgb_type)
+
+ model_metrics = "evalution is OFF, no metrics"
+
+ if xgb_type == 'C' or xgb_type == 'c':
+ categories = len(set(y_train))
+ plpy.info("xgboost_train_parallel() Classification categories: " + str(categories))
+
+ if categories < 2:
+ plpy.error("Wrong Xgboost Categories: " + str(categories))
+
+ gbm = xgb.XGBClassifier(**params_dict)
+
+ # Fit model
+ gbm.fit(X_train, y_train)
+
+ # 3) Compute and return model metrics score
+ if evaluation:
+ y_pred_test = gbm.predict(X_test)
+
+ if categories == 2:
+ scores = np.array(precision_recall_fscore_support(y_test, y_pred_test)).transpose()
+
+ # precision: TP / (TP + FP) where tp is the number of true positives and fp the number of false positives.
+ # recall: TP / (TP + FN) where tp is the number of true positives and fn the number of false negatives.
+ # fscore: F1=2PR/(P+R)
+ # support: is the number of occurrences of each class in y_true
+ model_metrics = pd.DataFrame(scores, columns=['precision', 'recall', 'f1score', 'support'])
+ model_metrics['class'] = gbm.classes_
+ model_metrics = model_metrics[['class'] + ['precision', 'recall', 'f1score', 'support']]
+
+ model_metrics = model_metrics.to_string(index=False)
+
+ else:
+ model_metrics = dict()
+ model_metrics['kappa'] = cohen_kappa_score(y_test, y_pred_test)
+ model_metrics['acc'] = accuracy_score(y_test, y_pred_test)
+ model_metrics = str(model_metrics)
+
+ # 1) save model_metric, 2) [x], 3) dump model, 4) current parameters
+ return model_metrics, features, pickle.dumps(gbm), str(params)
+
+ else:
+ gbm = xgb.XGBRegressor(**params_dict)
+ gbm.fit(X_train, y_train)
+
+ if evaluation:
+ # R2squared: coefficient of determination
+ metric_labels = ['mae', 'mse', 'R2squared', 'rmse']
+
+ y_test_pred = gbm.predict(X_test)
+ mae2 = mean_absolute_error(y_test_pred, y_test)
+ mse2 = mean_squared_error(y_test_pred, y_test)
+ r2 = r2_score(y_test_pred, y_test)
+ rmse2 = np.sqrt(mse2)
+
+ model_metrics = {'mae': mae2, 'mse': mse2, 'r2squared': r2, 'rmse': rmse2}
+ model_metrics = pd.DataFrame(model_metrics, index=list('1'))
+ model_metrics = model_metrics.to_string()
+
+ # 1) save model_metric, 2) [x], 3) dump model, 4) current parameters
+ return (model_metrics, features, pickle.dumps(gbm), params)
+
+
+def cartesian_product(params):
+ """
+ Expand a dict of parameters into a Cartesian product
+ For example, if the input is:
+ "{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': (5,6,7), 'objective': ('binary:logistic',)}"
+ Then, cartesian_product is:
+ [('booster = gbtree', 'eta = 0.1', 'max_depth = 5', 'objective = binary:logistic'),
+ ('booster = gbtree', 'eta = 0.1', 'max_depth = 6', 'objective = binary:logistic'),
+ ('booster = gbtree', 'eta = 0.1', 'max_depth = 7', 'objective = binary:logistic')]
+ """
+ # transfer params => Cartesian product
+ cp_list = []
+
+ for key, val in params.items():
+ if val and isinstance(val, collections.Iterable):
+ r = []
+ for v in val:
+ r.append("""{k} = {v}""".format(k=key, v=v))
+ else:
+ r = ["""{k} = {v}""".format(k=key, v=val)]
+ cp_list.append(r)
+ cartesian_product = [i for i in itertools.product(*cp_list)]
+ return cartesian_product
+
+
+def xgboost_grid_search(schema_madlib, input_table, grid_search_results_tbl, id_column,
+ dependent_varname, features_list, xgb_type, params_str, evaluation, verbose, **kwargs):
+ """
+ xgboost grid search main entry.
+ """
+
+ y_type = get_column_and_ret_type(schema_madlib, input_table, id_column, dependent_varname, features_list, verbose)
+
+ # Expand the grid-search parameters, Expand the params to run-grid search
+ if params_str is None or params_str is "" or ":" not in params_str:
+ params_str = "{}"
+ plpy.info("xgboost_grid_search() params_str type is: " + params_str)
+
+ # str => dict
+ params = ast.literal_eval(re.sub("[\\t]", "", params_str).strip())
+
+ # trans params => Cartesian product
+ params_grid = cartesian_product(params)
+
+ if verbose:
+ plpy.info("xgboost_grid_search() Cartesian product DONE: " + str(params_grid))
+
+ # Save each parameter list in the grid as a row in a distributed table
+ grid_search_params_temp_tbl = unique_string(desp='gridSearchXGB')
+ sql = """
+ DROP TABLE IF EXISTS {grid_search_params_temp_tbl};
+ CREATE TEMP TABLE {grid_search_params_temp_tbl} (id INT, params TEXT)
+ m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (id)')
+ m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `');
+ """.format(grid_search_params_temp_tbl=grid_search_params_temp_tbl)
+
+ if verbose:
+ plpy.info(sql)
+
+ plpy.execute(sql)
+
+ # insert grid search parameters into the g_s_p table
+ sql = """INSERT INTO {grid_search_params_temp_tbl} VALUES ({id}, $${val}$$);"""
+
+ for indx, val in enumerate(params_grid):
+ plpy.execute(
+ sql.format(
+ grid_search_params_temp_tbl=grid_search_params_temp_tbl,
+ id=indx + 1,
+ val=val
+ )
+ )
+
+ # Extract features from table and persist as serialized dataframe
+ col_list = features_list + [dependent_varname] # X + [y], quote_idented
+ sql = """
+ DROP TABLE IF EXISTS {grid_search_params_temp_tbl}_df;
+ CREATE TEMP TABLE {grid_search_params_temp_tbl}_df m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `')
+ AS
+ (
+ SELECT
+ 1 as id,
+ {schema_madlib}.__save_table_as_bytea__(
+ '{input_table}',
+ '{id_column}',
+ '{class_label}',
+ ARRAY[{features_list}],
+ {verbose}
+ ) as df
+ ) m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (id)');
+ """.format(
+ schema_madlib=schema_madlib,
+ grid_search_params_temp_tbl=grid_search_params_temp_tbl,
+ input_table=input_table,
+ id_column=id_column,
+ class_label=dependent_varname,
+ features_list=str(features_list).replace('[', '').replace(']', ''),
+ verbose=verbose
+ )
+
+ if verbose:
+ plpy.info(sql)
+
+ plpy.execute(sql)
+
+ # 5) Invoke XGBoost's train by passing each row from parameter list table. This will run in parallel.
+ sql = """
+ CREATE TABLE {grid_search_results_tbl} m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `')
+ AS (
+ SELECT
+ id,
+ now() as train_timestamp,
+ '{input_table}' as source_table,
+ '{y_type}'::TEXT as y_type,
+ (mdl_results).metrics,
+ (mdl_results).features,
+ (mdl_results).model,
+ (mdl_results).params
+ FROM (
+ SELECT
+ t1.id,
+ {schema_madlib}.__xgboost_train_parallel__(
+ df,
+ '{class_label}',
+ ARRAY[{features}],
+ '{xgb_type}',
+ params,
+ {evaluation},
+ {verbose}
+ ) AS mdl_results
+ FROM
+ {grid_search_params_temp_tbl} t1,
+ {grid_search_params_temp_tbl}_df
+ )q
+ );
+ """.format(
+ schema_madlib=schema_madlib,
+ y_type=y_type,
+ grid_search_results_tbl=grid_search_results_tbl,
+ grid_search_params_temp_tbl=grid_search_params_temp_tbl,
+ xgb_type=xgb_type,
+ features=str(features_list).replace('[', '').replace(']', '').replace(',', ','),
+ input_table=input_table,
+ class_label=dependent_varname,
+ evaluation=evaluation,
+ verbose=verbose
+ )
+
+ if verbose:
+ plpy.info("xgboost_grid_search() " + sql)
+
+ plpy.execute(sql)
+ return """Grid search results saved in {tbl}""".format(tbl=grid_search_results_tbl)
diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.sql_in b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.sql_in
new file mode 100644
index 0000000..ea71952
--- /dev/null
+++ b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.sql_in
@@ -0,0 +1,81 @@
+/*
+ * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd.
+ *
+ * openGauss is licensed under Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *
+ * http://license.coscl.org.cn/MulanPSL2
+ *
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * -------------------------------------------------------------------------
+ */
+
+m4_include(`SQLCommon.m4')
+
+----------------------------------------------------------------------------------------------
+------ xgboost grid search Algorithm
+----------------------------------------------------------------------------------------------
+
+------------------------------------------------
+-- Prepare Dataset BY pandas
+------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__save_table_as_bytea__(
+ input_table TEXT, /* dataset table */
+ id_column TEXT, /* id column */
+ dependent_varname TEXT, /* class label */
+ features_list text[], /* independent_v */
+ verbose BOOL
+)
+RETURNS bytea AS $$
+PythonFunction(xgboost_gs, xgboost_utilities, save_table_as_bytea)
+$$ LANGUAGE plpythonu;
+
+
+------------------------------------------------
+-- Type for internal train results
+------------------------------------------------
+DROP TYPE IF EXISTS MADLIB_SCHEMA.xgb_train_results_type cascade;
+CREATE TYPE MADLIB_SCHEMA.xgb_train_results_type AS(
+ metrics TEXT, /* save metrices info, e.g. mae, mse */
+ features TEXT[], /* save X */
+ model bytea, /* dump model */
+ params TEXT /* parameters used */
+);
+
+------------------------------------------------
+-- UDF to train XGBoost
+------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__xgboost_train_parallel__(
+ dframe bytea, /* df from pandas */
+ dependent_varname TEXT, /* class label */
+ features_list text[], /* independent_v */
+ xgb_type CHAR, /* Classifier or Regression */
+ params TEXT, /* special param */
+ evaluation BOOL, /* evaluation the model? default FALSE */
+ verbose BOOL
+)
+RETURNS MADLIB_SCHEMA.xgb_train_results_type AS $$
+PythonFunction(xgboost_gs, xgboost_utilities, xgboost_train_parallel)
+$$ LANGUAGE plpythonu;
+
+------------------------------------------------
+-- XGBoost grid search
+------------------------------------------------
+
+CREATE OR REPLACE function MADLIB_SCHEMA.xgboost_grid_search(
+ input_table TEXT,
+ grid_search_results_tbl TEXT,
+ id_column TEXT,
+ dependent_varname TEXT,
+ features_list TEXT[],
+ xgb_type CHAR, /* Classifier or Regressor */
+ params_str TEXT,
+ evaluation BOOL,
+ verbose BOOL
+)RETURNS TEXT AS $$
+PythonFunction(xgboost_gs, xgboost_utilities, xgboost_grid_search)
+$$ LANGUAGE plpythonu;
Reference in New Issue View Git Blame Copy Permalink